diff options
-rwxr-xr-x | jni/Android.mk | 6 | ||||
-rwxr-xr-x | jni/feature_mos/src/mosaic_renderer/FrameBuffer.cpp | 99 | ||||
-rwxr-xr-x | jni/feature_mos/src/mosaic_renderer/FrameBuffer.h | 31 | ||||
-rwxr-xr-x | jni/feature_mos/src/mosaic_renderer/WarpRenderer.cpp | 378 | ||||
-rwxr-xr-x | jni/feature_mos/src/mosaic_renderer/WarpRenderer.h | 80 | ||||
-rw-r--r-- | jni/feature_mos_jni.cpp | 93 | ||||
-rw-r--r-- | jni/mosaic_renderer_jni.cpp | 338 | ||||
-rw-r--r-- | jni/mosaic_renderer_jni.h | 23 | ||||
-rw-r--r-- | res/layout/pano_capture.xml | 23 | ||||
-rw-r--r-- | src/com/android/camera/panorama/CaptureView.java | 70 | ||||
-rw-r--r-- | src/com/android/camera/panorama/MosaicRenderer.java | 67 | ||||
-rw-r--r-- | src/com/android/camera/panorama/MosaicRendererSurfaceView.java | 322 | ||||
-rw-r--r-- | src/com/android/camera/panorama/MosaicRendererSurfaceViewRenderer.java | 55 | ||||
-rw-r--r-- | src/com/android/camera/panorama/PanoramaActivity.java | 11 |
14 files changed, 1505 insertions, 91 deletions
diff --git a/jni/Android.mk b/jni/Android.mk index 9d04bf5..d653573 100755 --- a/jni/Android.mk +++ b/jni/Android.mk @@ -13,6 +13,7 @@ LOCAL_CFLAGS := -O3 -DNDEBUG LOCAL_SRC_FILES := \ feature_mos_jni.cpp \ + mosaic_renderer_jni.cpp \ feature_mos/src/mosaic/trsMatrix.cpp \ feature_mos/src/mosaic/AlignFeatures.cpp \ feature_mos/src/mosaic/Blend.cpp \ @@ -20,6 +21,8 @@ LOCAL_SRC_FILES := \ feature_mos/src/mosaic/ImageUtils.cpp \ feature_mos/src/mosaic/Mosaic.cpp \ feature_mos/src/mosaic/Pyramid.cpp \ + feature_mos/src/mosaic_renderer/WarpRenderer.cpp \ + feature_mos/src/mosaic_renderer/FrameBuffer.cpp \ feature_stab/db_vlvm/db_feature_detection.cpp \ feature_stab/db_vlvm/db_feature_matching.cpp \ feature_stab/db_vlvm/db_framestitching.cpp \ @@ -34,7 +37,8 @@ LOCAL_SRC_FILES := \ feature_stab/src/dbreg/dbstabsmooth.cpp \ feature_stab/src/dbreg/vp_motionmodel.c -LOCAL_SHARED_LIBRARIES := liblog libnativehelper +LOCAL_SHARED_LIBRARIES := liblog libnativehelper libGLESv2 +#LOCAL_LDLIBS := -L$(SYSROOT)/usr/lib -ldl -llog -lGLESv2 -L$(TARGET_OUT) LOCAL_MODULE_TAGS := optional diff --git a/jni/feature_mos/src/mosaic_renderer/FrameBuffer.cpp b/jni/feature_mos/src/mosaic_renderer/FrameBuffer.cpp new file mode 100755 index 0000000..9a07e49 --- /dev/null +++ b/jni/feature_mos/src/mosaic_renderer/FrameBuffer.cpp @@ -0,0 +1,99 @@ +#include "FrameBuffer.h" + +FrameBuffer::FrameBuffer() +{ + Reset(); +} + +FrameBuffer::~FrameBuffer() { +} + +void FrameBuffer::Reset() { + mFrameBufferName = -1; + mTextureName = -1; + mWidth = 0; + mHeight = 0; + mFormat = -1; +} + +bool FrameBuffer::InitializeGLContext() { + Reset(); + return CreateBuffers(); +} + +bool FrameBuffer::Init(int width, int height, GLenum format) { + if (mFrameBufferName == (GLuint)-1) { + if (!CreateBuffers()) { + return false; + } + } + glBindFramebuffer(GL_FRAMEBUFFER, mFrameBufferName); + glBindTexture(GL_TEXTURE_2D, mTextureName); + + glTexImage2D(GL_TEXTURE_2D, + 0, + format, + width, + height, + 0, + format, + GL_UNSIGNED_BYTE, + NULL); + if (!checkGlError("bind/teximage")) { + return false; + } + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + // This is necessary to work with user-generated frame buffers with + // dimensions that are NOT powers of 2. + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + // Attach texture to frame buffer. + glFramebufferTexture2D(GL_FRAMEBUFFER, + GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, + mTextureName, + 0); + + if (!checkGlError("texture setup")) { + return false; + } + mWidth = width; + mHeight = height; + mFormat = format; + glBindFramebuffer(GL_FRAMEBUFFER, 0); + return true; +} + +bool FrameBuffer::CreateBuffers() { + glGenFramebuffers(1, &mFrameBufferName); + glGenTextures(1, &mTextureName); + if (!checkGlError("texture generation")) { + return false; + } + return true; +} + +GLuint FrameBuffer::GetTextureName() const { + return mTextureName; +} + +GLuint FrameBuffer::GetFrameBufferName() const { + return mFrameBufferName; +} + +GLenum FrameBuffer::GetFormat() const { + return mFormat; +} + +int FrameBuffer::GetWidth() const { + return mWidth; +} + +int FrameBuffer::GetHeight() const { + return mHeight; +} + + + diff --git a/jni/feature_mos/src/mosaic_renderer/FrameBuffer.h b/jni/feature_mos/src/mosaic_renderer/FrameBuffer.h new file mode 100755 index 0000000..b6a20ad --- /dev/null +++ b/jni/feature_mos/src/mosaic_renderer/FrameBuffer.h @@ -0,0 +1,31 @@ +#pragma once + +#include <EGL/egl.h> +#include <GLES2/gl2.h> +#include <GLES2/gl2ext.h> + +extern bool checkGlError(const char* op); + +class FrameBuffer { + public: + FrameBuffer(); + virtual ~FrameBuffer(); + + bool InitializeGLContext(); + bool Init(int width, int height, GLenum format); + GLuint GetTextureName() const; + GLuint GetFrameBufferName() const; + GLenum GetFormat() const; + + int GetWidth() const; + int GetHeight() const; + + private: + void Reset(); + bool CreateBuffers(); + GLuint mFrameBufferName; + GLuint mTextureName; + int mWidth; + int mHeight; + GLenum mFormat; +}; diff --git a/jni/feature_mos/src/mosaic_renderer/WarpRenderer.cpp b/jni/feature_mos/src/mosaic_renderer/WarpRenderer.cpp new file mode 100755 index 0000000..9274721 --- /dev/null +++ b/jni/feature_mos/src/mosaic_renderer/WarpRenderer.cpp @@ -0,0 +1,378 @@ +#include "WarpRenderer.h" + +#include <GLES2/gl2ext.h> + +#include <android/log.h> +#define LOG_TAG "WarpRenderer" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__) + + +static const char gVertexShader[] = +"uniform mat4 u_affinetrans; \n" +"uniform mat4 u_viewporttrans; \n" +"uniform mat4 u_scalingtrans; \n" +"attribute vec4 a_position; \n" +"attribute vec2 a_texCoord; \n" +"varying vec2 v_texCoord; \n" +"void main() \n" +"{ \n" +" gl_Position = u_scalingtrans * u_viewporttrans * u_affinetrans * a_position; \n" +" v_texCoord = a_texCoord; \n" +"} \n"; + +static const char gFragmentShader[] = +"precision mediump float; \n" +"varying vec2 v_texCoord; \n" +"uniform sampler2D s_texture; \n" +"void main() \n" +"{ \n" +" vec4 color; \n" +" color = texture2D(s_texture, v_texCoord); \n" +" gl_FragColor = color; \n" +"} \n"; + + +const GLfloat g_vVertices[] = { + -1.f, -1.f, 0.0f, 1.0f, // Position 0 + 0.0f, 1.0f, // TexCoord 0 + 1.f, -1.f, 0.0f, 1.0f, // Position 1 + 1.0f, 1.0f, // TexCoord 1 + -1.f, 1.f, 0.0f, 1.0f, // Position 2 + 0.0f, 0.0f, // TexCoord 2 + 1.f, 1.f, 0.0f, 1.0f, // Position 3 + 1.0f, 0.0f // TexCoord 3 +}; + +const int VERTEX_STRIDE = 6 * sizeof(GLfloat); + +GLushort g_iIndices[] = { 0, 1, 2, 3 }; + +WarpRenderer::WarpRenderer() + : mGlProgram(0), + mInputTextureName(-1), + mInputTextureWidth(0), + mInputTextureHeight(0), + mSurfaceWidth(0), + mSurfaceHeight(0) + { + InitializeGLContext(); +} + +WarpRenderer::~WarpRenderer() { +} + +GLuint WarpRenderer::loadShader(GLenum shaderType, const char* pSource) { + GLuint shader = glCreateShader(shaderType); + if (shader) { + glShaderSource(shader, 1, &pSource, NULL); + glCompileShader(shader); + GLint compiled = 0; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); + if (!compiled) { + GLint infoLen = 0; + glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen); + if (infoLen) { + char* buf = (char*) malloc(infoLen); + if (buf) { + glGetShaderInfoLog(shader, infoLen, NULL, buf); + LOGE("Could not compile shader %d:\n%s\n", + shaderType, buf); + free(buf); + } + glDeleteShader(shader); + shader = 0; + } + } + } + return shader; +} + +GLuint WarpRenderer::createProgram(const char* pVertexSource, const char* pFragmentSource) +{ + GLuint vertexShader = loadShader(GL_VERTEX_SHADER, pVertexSource); + if (!vertexShader) + { + return 0; + } + LOGI("VertexShader Loaded!"); + + GLuint pixelShader = loadShader(GL_FRAGMENT_SHADER, pFragmentSource); + if (!pixelShader) + { + return 0; + } + LOGI("FragmentShader Loaded!"); + + GLuint program = glCreateProgram(); + if (program) + { + glAttachShader(program, vertexShader); + checkGlError("glAttachShader"); + glAttachShader(program, pixelShader); + checkGlError("glAttachShader"); + + LOGI("Shaders Attached!"); + + glLinkProgram(program); + GLint linkStatus = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); + + LOGI("Program Linked!"); + + if (linkStatus != GL_TRUE) + { + GLint bufLength = 0; + glGetProgramiv(program, GL_INFO_LOG_LENGTH, &bufLength); + if (bufLength) + { + char* buf = (char*) malloc(bufLength); + if (buf) + { + glGetProgramInfoLog(program, bufLength, NULL, buf); + LOGE("Could not link program:\n%s\n", buf); + free(buf); + } + } + glDeleteProgram(program); + program = 0; + } + } + return program; +} + +bool WarpRenderer::InitializeGLProgram() +{ + bool succeeded = false; + do { + GLuint glProgram; + glProgram = createProgram(VertexShaderSource(), + FragmentShaderSource()); + if (!glProgram) { + break; + } + + glUseProgram(glProgram); + if (!checkGlError("glUseProgram")) break; + + // Get attribute locations + mPositionLoc = glGetAttribLocation(glProgram, "a_position"); + mAffinetransLoc = glGetUniformLocation(glProgram, "u_affinetrans"); + mViewporttransLoc = glGetUniformLocation(glProgram, "u_viewporttrans"); + mScalingtransLoc = glGetUniformLocation(glProgram, "u_scalingtrans"); + mTexCoordLoc = glGetAttribLocation(glProgram, "a_texCoord"); + + // Get sampler location + mSamplerLoc = glGetUniformLocation(glProgram, "s_texture"); + + mGlProgram = glProgram; + succeeded = true; + } while (false); + + if (!succeeded && (mGlProgram != 0)) + { + glDeleteProgram(mGlProgram); + checkGlError("glDeleteProgram"); + mGlProgram = 0; + } + return succeeded; +} + +void WarpRenderer::SetViewportMatrix(int w, int h, int W, int H) +{ + for(int i=0; i<16; i++) + { + mViewportMatrix[i] = 0.0f; + } + + mViewportMatrix[0] = float(w)/float(W); + mViewportMatrix[5] = float(h)/float(H); + mViewportMatrix[10] = 1.0f; + mViewportMatrix[12] = -1.0f + float(w)/float(W); + mViewportMatrix[13] = -1.0f + float(h)/float(H); + mViewportMatrix[15] = 1.0f; +} + +void WarpRenderer::SetScalingMatrix(float xscale, float yscale) +{ + for(int i=0; i<16; i++) + { + mScalingMatrix[i] = 0.0f; + } + + mScalingMatrix[0] = xscale; + mScalingMatrix[5] = yscale; + mScalingMatrix[10] = 1.0f; + mScalingMatrix[15] = 1.0f; +} + +// Set this renderer to use the default frame-buffer (screen) and +// set the viewport size to be the given width and height (pixels). +bool WarpRenderer::SetupGraphics(int width, int height) +{ + bool succeeded = false; + do { + if (mGlProgram == 0) + { + if (!InitializeGLProgram()) + { + break; + } + } + glUseProgram(mGlProgram); + if (!checkGlError("glUseProgram")) break; + + glBindFramebuffer(GL_FRAMEBUFFER, 0); + + mFrameBuffer = NULL; + mSurfaceWidth = width; + mSurfaceHeight = height; + + glViewport(0, 0, mSurfaceWidth, mSurfaceHeight); + if (!checkGlError("glViewport")) break; + succeeded = true; + } while (false); + + return succeeded; +} + + +// Set this renderer to use the specified FBO and +// set the viewport size to be the width and height of this FBO. +bool WarpRenderer::SetupGraphics(FrameBuffer* buffer) +{ + bool succeeded = false; + do { + if (mGlProgram == 0) + { + if (!InitializeGLProgram()) + { + break; + } + } + glUseProgram(mGlProgram); + if (!checkGlError("glUseProgram")) break; + + glBindFramebuffer(GL_FRAMEBUFFER, buffer->GetFrameBufferName()); + + mFrameBuffer = buffer; + mSurfaceWidth = mFrameBuffer->GetWidth(); + mSurfaceHeight = mFrameBuffer->GetHeight(); + + glViewport(0, 0, mSurfaceWidth, mSurfaceHeight); + if (!checkGlError("glViewport")) break; + succeeded = true; + } while (false); + + return succeeded; +} + +bool WarpRenderer::Clear(float r, float g, float b, float a) +{ + bool succeeded = false; + do { + bool rt = (mFrameBuffer == NULL)? + SetupGraphics(mSurfaceWidth, mSurfaceHeight) : + SetupGraphics(mFrameBuffer); + + if(!rt) + break; + + glClearColor(r, g, b, a); + glClear(GL_COLOR_BUFFER_BIT); + + succeeded = true; + } while (false); + return succeeded; + +} + +bool WarpRenderer::DrawTexture(GLfloat *affine) +{ + bool succeeded = false; + do { + bool rt = (mFrameBuffer == NULL)? + SetupGraphics(mSurfaceWidth, mSurfaceHeight) : + SetupGraphics(mFrameBuffer); + + if(!rt) + break; + + glDisable(GL_BLEND); + + glActiveTexture(GL_TEXTURE0); + if (!checkGlError("glActiveTexture")) break; + + const GLenum texture_type = InputTextureType(); + glBindTexture(texture_type, mInputTextureName); + if (!checkGlError("glBindTexture")) break; + + // Set the sampler texture unit to 0 + glUniform1i(mSamplerLoc, 0); + + // Load the vertex position + glVertexAttribPointer(mPositionLoc, 4, GL_FLOAT, + GL_FALSE, VERTEX_STRIDE, g_vVertices); + + // Load the texture coordinate + glVertexAttribPointer(mTexCoordLoc, 2, GL_FLOAT, + GL_FALSE, VERTEX_STRIDE, &g_vVertices[4]); + + glEnableVertexAttribArray(mPositionLoc); + glEnableVertexAttribArray(mTexCoordLoc); + + // pass matrix information to the vertex shader + glUniformMatrix4fv(mAffinetransLoc, 1, GL_FALSE, affine); + glUniformMatrix4fv(mViewporttransLoc, 1, GL_FALSE, mViewportMatrix); + glUniformMatrix4fv(mScalingtransLoc, 1, GL_FALSE, mScalingMatrix); + + // And, finally, execute the GL draw command. + glDrawElements(GL_TRIANGLE_STRIP, 4, GL_UNSIGNED_SHORT, g_iIndices); + + checkGlError("glDrawElements"); + + glBindFramebuffer(GL_FRAMEBUFFER, 0); + succeeded = true; + } while (false); + return succeeded; +} + +void WarpRenderer::InitializeGLContext() +{ + if(mFrameBuffer != NULL) + { + delete mFrameBuffer; + mFrameBuffer = NULL; + } + + mInputTextureName = -1; + mGlProgram = 0; + mTexHandle = 0; +} + +int WarpRenderer::GetTextureName() +{ + return mInputTextureName; +} + +void WarpRenderer::SetInputTextureName(GLuint textureName) +{ + mInputTextureName = textureName; +} + +void WarpRenderer::SetInputTextureDimensions(int width, int height) +{ + mInputTextureWidth = width; + mInputTextureHeight = height; +} + + +const char* WarpRenderer::VertexShaderSource() const +{ + return gVertexShader; +} + +const char* WarpRenderer::FragmentShaderSource() const +{ + return gFragmentShader; +} diff --git a/jni/feature_mos/src/mosaic_renderer/WarpRenderer.h b/jni/feature_mos/src/mosaic_renderer/WarpRenderer.h new file mode 100755 index 0000000..7986215 --- /dev/null +++ b/jni/feature_mos/src/mosaic_renderer/WarpRenderer.h @@ -0,0 +1,80 @@ +#pragma once + +#include "FrameBuffer.h" + +#include <GLES2/gl2.h> + +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +class WarpRenderer { + public: + WarpRenderer(); + virtual ~WarpRenderer(); + + // Initialize OpenGL resources + // @return true if successful + bool InitializeGLProgram(); + + bool SetupGraphics(FrameBuffer* buffer); + bool SetupGraphics(int width, int height); + + bool Clear(float r, float g, float b, float a); + + void SetViewportMatrix(int w, int h, int W, int H); + void SetScalingMatrix(float xscale, float yscale); + bool DrawTexture(GLfloat *affine); + + int GetTextureName(); + void SetInputTextureName(GLuint textureName); + void SetInputTextureDimensions(int width, int height); + + void InitializeGLContext(); + + protected: + + GLuint loadShader(GLenum shaderType, const char* pSource); + GLuint createProgram(const char*, const char* ); + + int SurfaceWidth() const { return mSurfaceWidth; } + int SurfaceHeight() const { return mSurfaceHeight; } + + private: + // Source code for shaders. + virtual const char* VertexShaderSource() const; + virtual const char* FragmentShaderSource() const; + + // Redefine this to use special texture types such as + // GL_TEXTURE_EXTERNAL_OES. + virtual GLenum InputTextureType() const { return GL_TEXTURE_2D; } + + + GLuint mGlProgram; + GLuint mInputTextureName; + int mInputTextureWidth; + int mInputTextureHeight; + + GLuint mTexHandle; // Handle to s_texture. + GLuint mTexCoordHandle; // Handle to a_texCoord. + GLuint mTriangleVerticesHandle; // Handle to vPosition. + + // Attribute locations + GLint mPositionLoc; + GLint mAffinetransLoc; + GLint mViewporttransLoc; + GLint mScalingtransLoc; + GLint mTexCoordLoc; + + GLfloat mViewportMatrix[16]; + GLfloat mScalingMatrix[16]; + + // Sampler location + GLint mSamplerLoc; + + int mSurfaceWidth; // Width of target surface. + int mSurfaceHeight; // Height of target surface. + + FrameBuffer *mFrameBuffer; +}; + diff --git a/jni/feature_mos_jni.cpp b/jni/feature_mos_jni.cpp index 577da24..c858728 100644 --- a/jni/feature_mos_jni.cpp +++ b/jni/feature_mos_jni.cpp @@ -47,8 +47,13 @@ extern "C" { #endif +#include "mosaic_renderer_jni.h" + char buffer[1024]; +double g_dAffinetrans[16]; +double g_dAffinetransInv[16]; + const int MAX_FRAMES_HR = 100; const int MAX_FRAMES_LR = 200; @@ -109,7 +114,8 @@ int Init(int mID, int nmax) t0 = now_ms(); - // When processing higher than 720x480 video, process low-res at quarter resolution + // When processing higher than 720x480 video, process low-res at + // quarter resolution if(tWidth[LR]>180) quarter_res[LR] = true; @@ -117,7 +123,8 @@ int Init(int mID, int nmax) // Check for initialization and if not, initialize if (!mosaic[mID]->isInitialized()) { - mosaic[mID]->initialize(blendingType, tWidth[mID], tHeight[mID], nmax, quarter_res[mID], thresh_still[mID]); + mosaic[mID]->initialize(blendingType, tWidth[mID], tHeight[mID], + nmax, quarter_res[mID], thresh_still[mID]); } t1 = now_ms(); @@ -126,7 +133,8 @@ int Init(int mID, int nmax) return 1; } -void GenerateQuarterResImagePlanar(ImageType im, int input_w, int input_h, ImageType &out) +void GenerateQuarterResImagePlanar(ImageType im, int input_w, int input_h, + ImageType &out) { ImageType imp; ImageType outp; @@ -246,7 +254,8 @@ void YUV420toYVU24(ImageType yvu24, ImageType yuv420sp, int width, int height) } } -void YUV420toYVU24_NEW(ImageType yvu24, ImageType yuv420sp, int width, int height) +void YUV420toYVU24_NEW(ImageType yvu24, ImageType yuv420sp, int width, + int height) { int frameSize = width * height; @@ -283,7 +292,8 @@ void YUV420toYVU24_NEW(ImageType yvu24, ImageType yuv420sp, int width, int heigh } -JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_allocateMosaicMemory(JNIEnv* env, jobject thiz, jint width, jint height) +JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_allocateMosaicMemory( + JNIEnv* env, jobject thiz, jint width, jint height) { tWidth[HR] = width; tHeight[HR] = height; @@ -300,8 +310,9 @@ JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_allocateMosaicMem tImage[HR][i] = ImageUtils::allocateImage(tWidth[HR], tHeight[HR], ImageUtils::IMAGE_TYPE_NUM_CHANNELS); } -} + AllocateTextureMemory(tWidth[LR], tHeight[LR]); +} JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_freeMosaicMemory( JNIEnv* env, jobject thiz) @@ -314,6 +325,45 @@ JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_freeMosaicMemory( { ImageUtils::freeImage(tImage[HR][i]); } + + FreeTextureMemory(); +} + + +void decodeYUV444SP(unsigned char* rgb, unsigned char* yuv420sp, int width, + int height) +{ + int frameSize = width * height; + + for (int j = 0, yp = 0; j < height; j++) + { + int vp = frameSize + j * width, u = 0, v = 0; + int up = vp + frameSize; + + for (int i = 0; i < width; i++, yp++, vp++, up++) + { + int y = (0xff & ((int) yuv420sp[yp])) - 16; + if (y < 0) y = 0; + + v = (0xff & yuv420sp[vp]) - 128; + u = (0xff & yuv420sp[up]) - 128; + + int y1192 = 1192 * y; + int r = (y1192 + 1634 * v); + int g = (y1192 - 833 * v - 400 * u); + int b = (y1192 + 2066 * u); + + if (r < 0) r = 0; else if (r > 262143) r = 262143; + if (g < 0) g = 0; else if (g > 262143) g = 262143; + if (b < 0) b = 0; else if (b > 262143) b = 262143; + + //rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff); + int p = j*width*3+i*3; + rgb[p+0] = (r<<6 & 0xFF0000)>>16; + rgb[p+1] = (g>>2 & 0xFF00)>>8; + rgb[p+2] = b>>10 & 0xFF; + } + } } @@ -349,6 +399,14 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI t0 = now_ms(); GenerateQuarterResImagePlanar(tImage[HR][frame_number_HR], tWidth[HR], tHeight[HR], tImage[LR][frame_number_LR]); + + + sem_wait(&gPreviewImageRGB_semaphore); + decodeYUV444SP(gPreviewImageRGB, tImage[LR][frame_number_LR], + gPreviewImageRGBWidth, gPreviewImageRGBHeight); + sem_post(&gPreviewImageRGB_semaphore); + + t1 = now_ms(); time_c = t1 - t0; LOGV("[%d] HR->LR [%d]: %g ms", frame_number_HR, frame_number_LR, @@ -369,6 +427,8 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI gTRS[0] = gTRS[4] = gTRS[8] = 1.0f; } + UpdateWarpTransformation(gTRS); + gTRS[9] = frame_number_HR; jfloatArray bytes = env->NewFloatArray(10); @@ -379,12 +439,14 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI return bytes; } -JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_setBlendingType(JNIEnv* env, jobject thiz, jint type) +JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_setBlendingType( + JNIEnv* env, jobject thiz, jint type) { blendingType = int(type); } -JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_reset(JNIEnv* env, jobject thiz) +JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_reset( + JNIEnv* env, jobject thiz) { frame_number_HR = 0; frame_number_LR = 0; @@ -392,7 +454,8 @@ JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_reset(JNIEnv* env Init(LR,MAX_FRAMES_LR); } -JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_createMosaic(JNIEnv* env, jobject thiz, jboolean value) +JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_createMosaic( + JNIEnv* env, jobject thiz, jboolean value) { high_res = bool(value); @@ -414,7 +477,8 @@ JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_createMosaic(JNIE } } -JNIEXPORT jintArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosaic(JNIEnv* env, jobject thiz) +JNIEXPORT jintArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosaic( + JNIEnv* env, jobject thiz) { int y,x; int width = mosaicWidth; @@ -422,7 +486,8 @@ JNIEXPORT jintArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosa int imageSize = width * height; // Convert back to RGB24 - resultBGR = ImageUtils::allocateImage(mosaicWidth, mosaicHeight, ImageUtils::IMAGE_TYPE_NUM_CHANNELS); + resultBGR = ImageUtils::allocateImage(mosaicWidth, mosaicHeight, + ImageUtils::IMAGE_TYPE_NUM_CHANNELS); ImageUtils::yvu2bgr(resultBGR, resultYVU, mosaicWidth, mosaicHeight); LOGV("MosBytes: %d, W = %d, H = %d", imageSize, width, height); @@ -434,7 +499,8 @@ JNIEXPORT jintArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosa { for(x=0; x<width; x++) { - image[y*width+x] = (0xFF<<24) | (resultBGR[y*width*3+x*3+2]<<16)| (resultBGR[y*width*3+x*3+1]<<8)| (resultBGR[y*width*3+x*3]); + image[y*width+x] = (0xFF<<24) | (resultBGR[y*width*3+x*3+2]<<16)| + (resultBGR[y*width*3+x*3+1]<<8)| (resultBGR[y*width*3+x*3]); } } @@ -456,7 +522,8 @@ JNIEXPORT jintArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosa return bytes; } -JNIEXPORT jbyteArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosaicNV21(JNIEnv* env, jobject thiz) +JNIEXPORT jbyteArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosaicNV21( + JNIEnv* env, jobject thiz) { int y,x; int width; diff --git a/jni/mosaic_renderer_jni.cpp b/jni/mosaic_renderer_jni.cpp new file mode 100644 index 0000000..21d3d39 --- /dev/null +++ b/jni/mosaic_renderer_jni.cpp @@ -0,0 +1,338 @@ +// OpenGL ES 2.0 code +#include <jni.h> +#include <android/log.h> + +#include <db_utilities_camera.h> +#include "mosaic/ImageUtils.h" +#include "mosaic_renderer/FrameBuffer.h" +#include "mosaic_renderer/WarpRenderer.h" +#include <stdio.h> +#include <stdlib.h> +#include <math.h> + +#include "mosaic_renderer_jni.h" + +#define LOG_TAG "MosaicRenderer" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__) + +// Texture handle +GLuint textureId[1]; + +bool warp_image = true; + +// Low-Res input image frame in RGB format for preview rendering +unsigned char* gPreviewImageRGB; +// Low-Res preview image width +int gPreviewImageRGBWidth; +// Low-Res preview image height +int gPreviewImageRGBHeight; + +// Semaphore to protect simultaneous read/writes from gPreviewImageRGB +sem_t gPreviewImageRGB_semaphore; + +// Off-screen preview FBO width (large enough to store the entire +// preview mosaic). +int gPreviewFBOWidth; +// Off-screen preview FBO height (large enough to store the entire +// preview mosaic). +int gPreviewFBOHeight; + +// Shader to add warped current frame to the preview FBO +WarpRenderer gWarper; +// Shader to warp and render the preview FBO to the screen +WarpRenderer gPreview; +// Off-screen FBO to store the result of gWarper +FrameBuffer *gBuffer; + +// Affine transformation in GL 4x4 format (column-major) to warp the +// current frame into the first frame coordinate system. +GLfloat g_dAffinetransGL[16]; + +// Affine transformation in GL 4x4 format (column-major) to warp the +// preview FBO into the current frame coordinate system. +GLfloat g_dAffinetransInvGL[16]; + +// GL 4x4 Identity transformation +GLfloat g_dAffinetransIdent[] = { + 1., 0., 0., 0., + 0., 1., 0., 0., + 0., 0., 1., 0., + 0., 0., 0., 1.}; + + +static void printGLString(const char *name, GLenum s) { + const char *v = (const char *) glGetString(s); + LOGI("GL %s = %s\n", name, v); +} + +// @return false if there was an error +bool checkGlError(const char* op) { + GLint error = glGetError(); + if (error != 0) { + LOGE("after %s() glError (0x%x)\n", op, error); + return false; + } + return true; +} + +void LoadTexture(unsigned char *buffer, int width, int height, GLuint texId) +{ + glBindTexture(GL_TEXTURE_2D, texId); + + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB, + GL_UNSIGNED_BYTE, buffer); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); +} + +void ReloadTexture(unsigned char *buffer, int width, int height, GLuint texId) +{ + glBindTexture(GL_TEXTURE_2D, texId); + + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_RGB, + GL_UNSIGNED_BYTE, buffer); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); +} + +void ConvertAffine3x3toGL4x4(double *matGL44, double *mat33) +{ + matGL44[0] = mat33[0]; + matGL44[1] = mat33[3]; + matGL44[2] = 0.0; + matGL44[3] = mat33[6]; + + matGL44[4] = mat33[1]; + matGL44[5] = mat33[4]; + matGL44[6] = 0.0; + matGL44[7] = mat33[7]; + + matGL44[8] = 0; + matGL44[9] = 0; + matGL44[10] = 1.0; + matGL44[11] = 0.0; + + matGL44[12] = mat33[2]; + matGL44[13] = mat33[5]; + matGL44[14] = 0.0; + matGL44[15] = mat33[8]; +} + +// This function computes fills the 4x4 matrices g_dAffinetrans and +// g_dAffinetransInv using the specified 3x3 affine transformation +// between the first captured frame and the current frame. The computed +// g_dAffinetrans is such that it warps the current frame into the +// coordinate system of the first frame. Thus, applying this transformation +// to each successive frame builds up the preview mosaic in the first frame +// coordinate system. Then the computed g_dAffinetransInv is such that it +// warps the computed preview mosaic into the coordinate system of the +// original (as captured) current frame. This has the effect of showing +// the current frame as is (without warping) but warping the rest of the +// mosaic data to still be in alignment with this frame. +void UpdateWarpTransformation(float *trs) +{ + double H[9], Hinv[9], Hp[9], Htemp[9]; + double K[9], Kinv[9]; + + int w = gPreviewImageRGBWidth; + int h = gPreviewImageRGBHeight; + + // K is the transformation to map the canonical [-1,1] vertex coordinate + // system to the [0,w] image coordinate system before applying the given + // affine transformation trs. + K[0] = w / 2.0; + K[1] = 0.0; + K[2] = w / 2.0; + K[3] = 0.0; + K[4] = -h / 2.0; + K[5] = h / 2.0; + K[6] = 0.0; + K[7] = 0.0; + K[8] = 1.0; + + db_Identity3x3(Kinv); + db_InvertCalibrationMatrix(Kinv, K); + + for(int i=0; i<9; i++) + { + H[i] = trs[i]; + } + + // Move the origin such that the frame is centered in the previewFBO + H[2] += (gPreviewFBOWidth / 2 - gPreviewImageRGBWidth / 2); + H[5] -= (gPreviewFBOHeight / 2 - gPreviewImageRGBHeight / 2); + + // Hp = inv(K) * H * K + db_Identity3x3(Htemp); + db_Multiply3x3_3x3(Htemp, H, K); + db_Multiply3x3_3x3(Hp, Kinv, Htemp); + + ConvertAffine3x3toGL4x4(g_dAffinetrans, Hp); + + //////////////////////////////////////////////// + ////// Compute g_dAffinetransInv now... ////// + //////////////////////////////////////////////// + + w = gPreviewFBOWidth; + h = gPreviewFBOHeight; + + K[0] = w / 2.0; + K[1] = 0.0; + K[2] = w / 2.0; + K[3] = 0.0; + K[4] = h / 2.0; + K[5] = h / 2.0; + K[6] = 0.0; + K[7] = 0.0; + K[8] = 1.0; + + db_Identity3x3(Kinv); + db_InvertCalibrationMatrix(Kinv, K); + + db_Identity3x3(Hinv); + db_InvertAffineTransform(Hinv, H); + + Hinv[2] += (gPreviewFBOWidth / 2 - gPreviewImageRGBWidth / 2); + Hinv[5] -= (gPreviewFBOHeight / 2 - gPreviewImageRGBHeight / 2); + + // Hp = inv(K) * Hinv * K + db_Identity3x3(Htemp); + db_Multiply3x3_3x3(Htemp, Hinv, K); + db_Multiply3x3_3x3(Hp, Kinv, Htemp); + + ConvertAffine3x3toGL4x4(g_dAffinetransInv, Hp); +} + +void AllocateTextureMemory(int width, int height) +{ + gPreviewImageRGBWidth = width; + gPreviewImageRGBHeight = height; + + sem_init(&gPreviewImageRGB_semaphore, 0, 1); + + sem_wait(&gPreviewImageRGB_semaphore); + gPreviewImageRGB = ImageUtils::allocateImage(gPreviewImageRGBWidth, + gPreviewImageRGBHeight, ImageUtils::IMAGE_TYPE_NUM_CHANNELS); + memset(gPreviewImageRGB, 0, gPreviewImageRGBWidth * + gPreviewImageRGBHeight * 3 * sizeof(unsigned char)); + sem_post(&gPreviewImageRGB_semaphore); + + gPreviewFBOWidth = PREVIEW_FBO_WIDTH_SCALE * gPreviewImageRGBWidth; + gPreviewFBOHeight = PREVIEW_FBO_HEIGHT_SCALE * gPreviewImageRGBHeight; + + UpdateWarpTransformation(g_dAffinetransIdent); +} + +void FreeTextureMemory() +{ + sem_wait(&gPreviewImageRGB_semaphore); + ImageUtils::freeImage(gPreviewImageRGB); + sem_post(&gPreviewImageRGB_semaphore); + + sem_destroy(&gPreviewImageRGB_semaphore); +} + +extern "C" +{ + JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_init( + JNIEnv * env, jobject obj); + JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_reset( + JNIEnv * env, jobject obj, jint width, jint height); + JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_step( + JNIEnv * env, jobject obj); + JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_ready( + JNIEnv * env, jobject obj); + JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_togglewarping( + JNIEnv * env, jobject obj); +}; + +JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_init( + JNIEnv * env, jobject obj) +{ + gWarper.InitializeGLProgram(); + gPreview.InitializeGLProgram(); + + glBindFramebuffer(GL_FRAMEBUFFER, 0); + glGenTextures(1, &textureId[0]); +} + +JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_reset( + JNIEnv * env, jobject obj, jint width, jint height) +{ + gBuffer = new FrameBuffer(); + gBuffer->Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA); + + sem_wait(&gPreviewImageRGB_semaphore); + memset(gPreviewImageRGB, 0, gPreviewImageRGBWidth * + gPreviewImageRGBHeight * 3 * sizeof(unsigned char)); + sem_post(&gPreviewImageRGB_semaphore); + + // Load texture + LoadTexture(gPreviewImageRGB, gPreviewImageRGBWidth, + gPreviewImageRGBHeight, textureId[0]); + + gWarper.SetupGraphics(gBuffer); + gWarper.Clear(0.0, 0.0, 0.0, 1.0); + gWarper.SetViewportMatrix(gPreviewImageRGBWidth, + gPreviewImageRGBHeight, gBuffer->GetWidth(), gBuffer->GetHeight()); + gWarper.SetScalingMatrix(1.0f, 1.0f); + gWarper.SetInputTextureName(textureId[0]); + + gPreview.SetupGraphics(width, height); + gPreview.Clear(0.0, 0.0, 0.0, 1.0); + gPreview.SetViewportMatrix(1, 1, 1, 1); + gPreview.SetScalingMatrix(1.0f, -1.0f); + gPreview.SetInputTextureName(gBuffer->GetTextureName()); +} + +JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_step( + JNIEnv * env, jobject obj) +{ + // Use the gWarper shader to apply the current frame transformation to the + // current frame and then add it to the gBuffer FBO. + gWarper.DrawTexture(g_dAffinetransGL); + + // Clear the screen to black. + glBindFramebuffer(GL_FRAMEBUFFER, 0); + glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + + // Use the gPreview shader to apply the inverse of the current frame + // transformation to the gBuffer FBO and render it to the screen. + gPreview.DrawTexture(g_dAffinetransInvGL); +} + +JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_togglewarping( + JNIEnv * env, jobject obj) +{ + warp_image = !warp_image; +} + + +JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_ready( + JNIEnv * env, jobject obj) +{ + sem_wait(&gPreviewImageRGB_semaphore); + ReloadTexture(gPreviewImageRGB, gPreviewImageRGBWidth, + gPreviewImageRGBHeight, textureId[0]); + sem_post(&gPreviewImageRGB_semaphore); + + if(!warp_image) + { + for(int i=0; i<16; i++) + { + g_dAffinetrans[i] = g_dAffinetransIdent[i]; + } + } + + for(int i=0; i<16; i++) + { + g_dAffinetransGL[i] = g_dAffinetrans[i]; + g_dAffinetransInvGL[i] = g_dAffinetransInv[i]; + } +} diff --git a/jni/mosaic_renderer_jni.h b/jni/mosaic_renderer_jni.h new file mode 100644 index 0000000..ed4180f --- /dev/null +++ b/jni/mosaic_renderer_jni.h @@ -0,0 +1,23 @@ +#pragma once +#include <GLES2/gl2.h> +#include <GLES2/gl2ext.h> +#include <semaphore.h> + +// The Preview FBO dimensions are determined from the low-res +// frame dimensions (gPreviewImageRGBWidth, gPreviewImageRGBHeight) +// using the scale factors below. +const int PREVIEW_FBO_WIDTH_SCALE = 4; +const int PREVIEW_FBO_HEIGHT_SCALE = 2; + +extern "C" void AllocateTextureMemory(int width, int height); +extern "C" void FreeTextureMemory(); +extern "C" void UpdateWarpTransformation(float *trs); + +extern unsigned char* gPreviewImageRGB; +extern int gPreviewImageRGBWidth; +extern int gPreviewImageRGBHeight; + +extern sem_t gPreviewImageRGB_semaphore; + +extern double g_dAffinetrans[16]; +extern double g_dAffinetransInv[16]; diff --git a/res/layout/pano_capture.xml b/res/layout/pano_capture.xml index 8b353fd..a12e6e6 100644 --- a/res/layout/pano_capture.xml +++ b/res/layout/pano_capture.xml @@ -15,19 +15,22 @@ --> <RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android" - android:id="@+id/pano_capture_layout" - android:visibility="invisible" - android:orientation="vertical" - android:layout_width="match_parent" - android:layout_height="match_parent"> - - <com.android.camera.panorama.CaptureView android:id="@+id/pano_capture_view" - android:layout_width="match_parent" + android:id="@+id/pano_capture_layout" android:visibility="invisible" + android:orientation="vertical" android:layout_width="match_parent" + android:layout_height="match_parent"> + + <FrameLayout android:layout_weight="1" android:gravity="center" + android:layout_width="match_parent" android:layout_height="match_parent"> + <com.android.camera.panorama.CaptureView + android:id="@+id/pano_capture_view" android:layout_width="match_parent" android:layout_height="match_parent" /> + <com.android.camera.panorama.MosaicRendererSurfaceView + android:id="@+id/pano_renderer" android:layout_gravity="center" + android:layout_width="960dp" android:layout_height="360dp" /> + </FrameLayout> <ImageView style="@style/PanoViewHorizontalGrayBar" - android:layout_alignParentTop="true" - android:layout_width="match_parent" /> + android:layout_alignParentTop="true" android:layout_width="match_parent" /> <LinearLayout android:id="@+id/pano_capture_control" style="@style/PanoViewHorizontalGrayBar" diff --git a/src/com/android/camera/panorama/CaptureView.java b/src/com/android/camera/panorama/CaptureView.java index 005c2a0..37f46bb 100644 --- a/src/com/android/camera/panorama/CaptureView.java +++ b/src/com/android/camera/panorama/CaptureView.java @@ -17,10 +17,8 @@ package com.android.camera.panorama; import android.content.Context; -import android.graphics.Bitmap; import android.graphics.Canvas; import android.graphics.Color; -import android.graphics.Matrix; import android.graphics.Paint; import android.graphics.Paint.Align; import android.graphics.RectF; @@ -31,30 +29,15 @@ import android.view.View; class CaptureView extends View { private static final String TAG = "CaptureView"; - - private Canvas mCanvas; - private Bitmap mCanvasBitmap; private String mStatusText = ""; private int mStartAngle = 0; private int mSweepAngle = 0; private int mWidth; private int mHeight; - private Bitmap mBitmap = null; - private Matrix mM = null; - private Matrix mMLast = null; private final Paint mPaint = new Paint(); - // Origin of the coordinate for appending a new alpha bitmap. - // mCanvasBitmap is 2000x2000, but the origin is set to (800, 800). - // All the alpha bitmaps grow from this origin. - float mAlphaOriginX; - float mAlphaOriginY; - public CaptureView(Context context, AttributeSet attrs) { super(context, attrs); - mM = new Matrix(); - mMLast = new Matrix(); - mMLast.reset(); mPaint.setStyle(Paint.Style.FILL); mPaint.setColor(Color.RED); mPaint.setAntiAlias(true); @@ -64,34 +47,13 @@ class CaptureView extends View { } protected void onSizeChanged(int w, int h, int oldw, int oldh) { - if (mCanvasBitmap != null) { - mCanvasBitmap.recycle(); - } Log.v(TAG, "onSizeChanged: W = " + w + ", H = " + h); - // TODO: 2000x1000 is a temporary setting from SRI's code. Should be fixed once the code is - // refactored. - mCanvasBitmap = Bitmap.createBitmap(2000, 1000, Bitmap.Config.ARGB_8888); - mCanvas = new Canvas(); - mCanvas.setBitmap(mCanvasBitmap); - mAlphaOriginX = mCanvasBitmap.getWidth() * 0.4f; - mAlphaOriginY = mCanvasBitmap.getHeight() * 0.4f; } public void onResume() { - if (mCanvasBitmap == null) { - mCanvasBitmap = Bitmap.createBitmap(2000, 1000, Bitmap.Config.ARGB_8888); - mCanvas = new Canvas(); - mCanvas.setBitmap(mCanvasBitmap); - mAlphaOriginX = mCanvasBitmap.getWidth() * 0.4f; - mAlphaOriginY = mCanvasBitmap.getHeight() * 0.4f; - } } public void onPause() { - if (mCanvasBitmap != null) { - mCanvasBitmap.recycle(); - mCanvasBitmap = null; - } } public void setStartAngle(int angle) { @@ -106,37 +68,15 @@ class CaptureView extends View { mStatusText = text; } - public void setBitmap(Bitmap bitmap, Matrix m) { - mBitmap = bitmap; - mM = m; - } - - public void setBitmap(Bitmap bitmap) { - mBitmap = bitmap; - } - @Override protected void onDraw(Canvas canvas) { mWidth = getWidth(); mHeight = getHeight(); - // Draw bitmaps according to the calculated panorama transformation. - if (mBitmap != null) { - mM.postTranslate(mAlphaOriginX, mAlphaOriginY); - mCanvas.drawBitmap(mBitmap, mM, mPaint); - - Matrix mInverse = mM; - mM.invert(mInverse); - mInverse.postTranslate(mWidth / 2 - mBitmap.getWidth() / 2, - mHeight / 2 - mBitmap.getHeight() / 2); - - canvas.drawBitmap(mCanvasBitmap, mInverse, mPaint); - - RectF rect = new RectF(mWidth / 2 - 100, 3 * mHeight / 4, - mWidth / 2 + 100, 3 * mHeight / 4 + 200); - canvas.drawText(mStatusText, mWidth / 2, mHeight / 2, mPaint); - canvas.drawArc(rect, -90 + mStartAngle, mSweepAngle, true, mPaint); - canvas.drawArc(rect, -90 - mStartAngle, mSweepAngle > 0 ? 2 : 0, true, mPaint); - } + RectF rect = new RectF(mWidth / 2 - 100, 3 * mHeight / 4, + mWidth / 2 + 100, 3 * mHeight / 4 + 200); + canvas.drawText(mStatusText, mWidth / 2, mHeight / 2, mPaint); + canvas.drawArc(rect, -90 + mStartAngle, mSweepAngle, true, mPaint); + canvas.drawArc(rect, -90 - mStartAngle, mSweepAngle > 0 ? 2 : 0, true, mPaint); } } diff --git a/src/com/android/camera/panorama/MosaicRenderer.java b/src/com/android/camera/panorama/MosaicRenderer.java new file mode 100644 index 0000000..8db95c0 --- /dev/null +++ b/src/com/android/camera/panorama/MosaicRenderer.java @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.camera.panorama; + +/** + * The Java interface to JNI calls regarding mosaic preview rendering. + * + */ +public class MosaicRenderer +{ + static + { + System.loadLibrary("jni_mosaic"); + } + + /** + * Function to be called in onSurfaceCreated() to initialize + * the GL context, load and link the shaders and create the + * program. + */ + public static native void init(); + + /** + * Pass the drawing surface's width and height to initialize the + * renderer viewports and FBO dimensions. + * + * @param width width of the drawing surface in pixels. + * @param height height of the drawing surface in pixels. + */ + public static native void reset(int width, int height); + + /** + * Function to be called in onDrawFrame() to update the screen with + * the new frame data. + */ + public static native void step(); + + /** + * Call this function when a new low-res frame has been processed by + * the mosaicing library. This will tell the renderer library to + * update its texture and warping transformation. Any calls to step() + * after this call will use the new image frame and transformation data. + */ + public static native void ready(); + + /** + * This function allows toggling between showing the input image data + * (without applying any warp) and the warped image data. This is more + * for debugging purposes to see if the image data is being updated + * correctly or not. + */ + public static native void togglewarping(); +} diff --git a/src/com/android/camera/panorama/MosaicRendererSurfaceView.java b/src/com/android/camera/panorama/MosaicRendererSurfaceView.java new file mode 100644 index 0000000..3220b22 --- /dev/null +++ b/src/com/android/camera/panorama/MosaicRendererSurfaceView.java @@ -0,0 +1,322 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.camera.panorama; + +import javax.microedition.khronos.egl.EGL10; +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.egl.EGLContext; +import javax.microedition.khronos.egl.EGLDisplay; + +import android.content.Context; +import android.graphics.PixelFormat; +import android.opengl.GLSurfaceView; +import android.util.AttributeSet; +import android.util.Log; + +public class MosaicRendererSurfaceView extends GLSurfaceView { + private static String TAG = "MosaicRendererSurfaceView"; + private static final boolean DEBUG = false; + private MosaicRendererSurfaceViewRenderer mRenderer; + + public MosaicRendererSurfaceView(Context context) { + super(context); + init(false, 0, 0); + setZOrderMediaOverlay(true); + } + + public MosaicRendererSurfaceView(Context context, AttributeSet attrs) { + super(context, attrs); + init(false, 0, 0); + setZOrderMediaOverlay(true); + } + + public MosaicRendererSurfaceView(Context context, boolean translucent, int depth, int stencil) { + super(context); + init(translucent, depth, stencil); + setZOrderMediaOverlay(true); + } + + + private void init(boolean translucent, int depth, int stencil) { + + /* By default, GLSurfaceView() creates a RGB_565 opaque surface. + * If we want a translucent one, we should change the surface's + * format here, using PixelFormat.TRANSLUCENT for GL Surfaces + * is interpreted as any 32-bit surface with alpha by SurfaceFlinger. + */ + if (translucent) { + this.getHolder().setFormat(PixelFormat.TRANSLUCENT); + } + + /* Setup the context factory for 2.0 rendering. + * See ContextFactory class definition below + */ + setEGLContextFactory(new ContextFactory()); + + /* We need to choose an EGLConfig that matches the format of + * our surface exactly. This is going to be done in our + * custom config chooser. See ConfigChooser class definition + * below. + */ + setEGLConfigChooser( translucent ? + new ConfigChooser(8, 8, 8, 8, depth, stencil) : + new ConfigChooser(5, 6, 5, 0, depth, stencil) ); + + /* Set the renderer responsible for frame rendering */ + mRenderer = new MosaicRendererSurfaceViewRenderer(); + setRenderer(mRenderer); + setRenderMode(RENDERMODE_WHEN_DIRTY); + } + + private static class ContextFactory implements GLSurfaceView.EGLContextFactory { + private static int EGL_CONTEXT_CLIENT_VERSION = 0x3098; + public EGLContext createContext(EGL10 egl, EGLDisplay display, EGLConfig eglConfig) { + Log.w(TAG, "creating OpenGL ES 2.0 context"); + checkEglError("Before eglCreateContext", egl); + int[] attrib_list = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL10.EGL_NONE }; + EGLContext context = egl.eglCreateContext(display, eglConfig, EGL10.EGL_NO_CONTEXT, attrib_list); + checkEglError("After eglCreateContext", egl); + return context; + } + + public void destroyContext(EGL10 egl, EGLDisplay display, EGLContext context) { + egl.eglDestroyContext(display, context); + } + } + + private static void checkEglError(String prompt, EGL10 egl) { + int error; + while ((error = egl.eglGetError()) != EGL10.EGL_SUCCESS) { + Log.e(TAG, String.format("%s: EGL error: 0x%x", prompt, error)); + } + } + + private static class ConfigChooser implements GLSurfaceView.EGLConfigChooser { + + public ConfigChooser(int r, int g, int b, int a, int depth, int stencil) { + mRedSize = r; + mGreenSize = g; + mBlueSize = b; + mAlphaSize = a; + mDepthSize = depth; + mStencilSize = stencil; + } + + /* This EGL config specification is used to specify 2.0 rendering. + * We use a minimum size of 4 bits for red/green/blue, but will + * perform actual matching in chooseConfig() below. + */ + private static final int EGL_OPENGL_ES2_BIT = 4; + private static final int[] CONFIG_ATTRIBUTES = + { + EGL10.EGL_RED_SIZE, 4, + EGL10.EGL_GREEN_SIZE, 4, + EGL10.EGL_BLUE_SIZE, 4, + EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL10.EGL_NONE + }; + + public EGLConfig chooseConfig(EGL10 egl, EGLDisplay display) { + + /* Get the number of minimally matching EGL configurations + */ + int[] num_config = new int[1]; + egl.eglChooseConfig(display, CONFIG_ATTRIBUTES, null, 0, num_config); + + int numConfigs = num_config[0]; + + if (numConfigs <= 0) { + throw new IllegalArgumentException("No configs match configSpec"); + } + + /* Allocate then read the array of minimally matching EGL configs + */ + EGLConfig[] configs = new EGLConfig[numConfigs]; + egl.eglChooseConfig(display, CONFIG_ATTRIBUTES, configs, numConfigs, num_config); + + if (DEBUG) { + printConfigs(egl, display, configs); + } + /* Now return the "best" one + */ + return chooseConfig(egl, display, configs); + } + + public EGLConfig chooseConfig(EGL10 egl, EGLDisplay display, + EGLConfig[] configs) { + for(EGLConfig config : configs) { + int d = findConfigAttrib(egl, display, config, + EGL10.EGL_DEPTH_SIZE, 0); + int s = findConfigAttrib(egl, display, config, + EGL10.EGL_STENCIL_SIZE, 0); + + // We need at least mDepthSize and mStencilSize bits + if (d < mDepthSize || s < mStencilSize) + continue; + + // We want an *exact* match for red/green/blue/alpha + int r = findConfigAttrib(egl, display, config, + EGL10.EGL_RED_SIZE, 0); + int g = findConfigAttrib(egl, display, config, + EGL10.EGL_GREEN_SIZE, 0); + int b = findConfigAttrib(egl, display, config, + EGL10.EGL_BLUE_SIZE, 0); + int a = findConfigAttrib(egl, display, config, + EGL10.EGL_ALPHA_SIZE, 0); + + if (r == mRedSize && g == mGreenSize && b == mBlueSize && a == mAlphaSize) + return config; + } + return null; + } + + private int findConfigAttrib(EGL10 egl, EGLDisplay display, + EGLConfig config, int attribute, int defaultValue) { + + if (egl.eglGetConfigAttrib(display, config, attribute, mValue)) { + return mValue[0]; + } + return defaultValue; + } + + private void printConfigs(EGL10 egl, EGLDisplay display, + EGLConfig[] configs) { + int numConfigs = configs.length; + Log.w(TAG, String.format("%d configurations", numConfigs)); + for (int i = 0; i < numConfigs; i++) { + Log.w(TAG, String.format("Configuration %d:\n", i)); + printConfig(egl, display, configs[i]); + } + } + + private void printConfig(EGL10 egl, EGLDisplay display, + EGLConfig config) { + int[] attributes = { + EGL10.EGL_BUFFER_SIZE, + EGL10.EGL_ALPHA_SIZE, + EGL10.EGL_BLUE_SIZE, + EGL10.EGL_GREEN_SIZE, + EGL10.EGL_RED_SIZE, + EGL10.EGL_DEPTH_SIZE, + EGL10.EGL_STENCIL_SIZE, + EGL10.EGL_CONFIG_CAVEAT, + EGL10.EGL_CONFIG_ID, + EGL10.EGL_LEVEL, + EGL10.EGL_MAX_PBUFFER_HEIGHT, + EGL10.EGL_MAX_PBUFFER_PIXELS, + EGL10.EGL_MAX_PBUFFER_WIDTH, + EGL10.EGL_NATIVE_RENDERABLE, + EGL10.EGL_NATIVE_VISUAL_ID, + EGL10.EGL_NATIVE_VISUAL_TYPE, + 0x3030, // EGL10.EGL_PRESERVED_RESOURCES, + EGL10.EGL_SAMPLES, + EGL10.EGL_SAMPLE_BUFFERS, + EGL10.EGL_SURFACE_TYPE, + EGL10.EGL_TRANSPARENT_TYPE, + EGL10.EGL_TRANSPARENT_RED_VALUE, + EGL10.EGL_TRANSPARENT_GREEN_VALUE, + EGL10.EGL_TRANSPARENT_BLUE_VALUE, + 0x3039, // EGL10.EGL_BIND_TO_TEXTURE_RGB, + 0x303A, // EGL10.EGL_BIND_TO_TEXTURE_RGBA, + 0x303B, // EGL10.EGL_MIN_SWAP_INTERVAL, + 0x303C, // EGL10.EGL_MAX_SWAP_INTERVAL, + EGL10.EGL_LUMINANCE_SIZE, + EGL10.EGL_ALPHA_MASK_SIZE, + EGL10.EGL_COLOR_BUFFER_TYPE, + EGL10.EGL_RENDERABLE_TYPE, + 0x3042 // EGL10.EGL_CONFORMANT + }; + String[] names = { + "EGL_BUFFER_SIZE", + "EGL_ALPHA_SIZE", + "EGL_BLUE_SIZE", + "EGL_GREEN_SIZE", + "EGL_RED_SIZE", + "EGL_DEPTH_SIZE", + "EGL_STENCIL_SIZE", + "EGL_CONFIG_CAVEAT", + "EGL_CONFIG_ID", + "EGL_LEVEL", + "EGL_MAX_PBUFFER_HEIGHT", + "EGL_MAX_PBUFFER_PIXELS", + "EGL_MAX_PBUFFER_WIDTH", + "EGL_NATIVE_RENDERABLE", + "EGL_NATIVE_VISUAL_ID", + "EGL_NATIVE_VISUAL_TYPE", + "EGL_PRESERVED_RESOURCES", + "EGL_SAMPLES", + "EGL_SAMPLE_BUFFERS", + "EGL_SURFACE_TYPE", + "EGL_TRANSPARENT_TYPE", + "EGL_TRANSPARENT_RED_VALUE", + "EGL_TRANSPARENT_GREEN_VALUE", + "EGL_TRANSPARENT_BLUE_VALUE", + "EGL_BIND_TO_TEXTURE_RGB", + "EGL_BIND_TO_TEXTURE_RGBA", + "EGL_MIN_SWAP_INTERVAL", + "EGL_MAX_SWAP_INTERVAL", + "EGL_LUMINANCE_SIZE", + "EGL_ALPHA_MASK_SIZE", + "EGL_COLOR_BUFFER_TYPE", + "EGL_RENDERABLE_TYPE", + "EGL_CONFORMANT" + }; + int[] value = new int[1]; + for (int i = 0; i < attributes.length; i++) { + int attribute = attributes[i]; + String name = names[i]; + if ( egl.eglGetConfigAttrib(display, config, attribute, value)) { + Log.w(TAG, String.format(" %s: %d\n", name, value[0])); + } else { + // Log.w(TAG, String.format(" %s: failed\n", name)); + while (egl.eglGetError() != EGL10.EGL_SUCCESS); + } + } + } + + // Subclasses can adjust these values: + protected int mRedSize; + protected int mGreenSize; + protected int mBlueSize; + protected int mAlphaSize; + protected int mDepthSize; + protected int mStencilSize; + private int[] mValue = new int[1]; + } + + public void setReady() + { + queueEvent(new Runnable() { + + @Override + public void run() { + mRenderer.setReady(); + } + }); + } + + public void toggleWarping() + { + queueEvent(new Runnable() { + + @Override + public void run() { + mRenderer.toggleWarping(); + } + }); + } +} diff --git a/src/com/android/camera/panorama/MosaicRendererSurfaceViewRenderer.java b/src/com/android/camera/panorama/MosaicRendererSurfaceViewRenderer.java new file mode 100644 index 0000000..a27d4be --- /dev/null +++ b/src/com/android/camera/panorama/MosaicRendererSurfaceViewRenderer.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.camera.panorama; + +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.opengles.GL10; + +import android.opengl.GLSurfaceView; + + +public class MosaicRendererSurfaceViewRenderer implements GLSurfaceView.Renderer +{ + @Override + public void onDrawFrame(GL10 gl) + { + MosaicRenderer.step(); + } + + @Override + public void onSurfaceChanged(GL10 gl, int width, int height) + { + MosaicRenderer.reset(width, height); + } + + @Override + public void onSurfaceCreated(GL10 gl, EGLConfig config) + { + MosaicRenderer.init(); + } + + public void setReady() + { + MosaicRenderer.ready(); + } + + public void toggleWarping() + { + MosaicRenderer.togglewarping(); + } + +} diff --git a/src/com/android/camera/panorama/PanoramaActivity.java b/src/com/android/camera/panorama/PanoramaActivity.java index 7b2042e..b66a311 100644 --- a/src/com/android/camera/panorama/PanoramaActivity.java +++ b/src/com/android/camera/panorama/PanoramaActivity.java @@ -79,6 +79,7 @@ public class PanoramaActivity extends Activity implements private SurfaceView mPreview; private ImageView mReview; private CaptureView mCaptureView; + private MosaicRendererSurfaceView mRealTimeMosaicView; private ShutterButton mShutterButton; private Button mStopButton; @@ -289,7 +290,10 @@ public class PanoramaActivity extends Activity implements private void updateProgress(float translationRate, int traversedAngleX, int traversedAngleY, Bitmap lowResBitmapAlpha, Matrix transformationMatrix) { - mCaptureView.setBitmap(lowResBitmapAlpha, transformationMatrix); + + mRealTimeMosaicView.setReady(); + mRealTimeMosaicView.requestRender(); + if (translationRate > 150) { // TODO: remove the text and draw implications according to the UI // spec. @@ -315,6 +319,8 @@ public class PanoramaActivity extends Activity implements mCaptureView.setStartAngle(-DEFAULT_SWEEP_ANGLE / 2); mReview = (ImageView) findViewById(R.id.pano_reviewarea); + mRealTimeMosaicView = (MosaicRendererSurfaceView) findViewById(R.id.pano_renderer); + mShutterButton = (ShutterButton) findViewById(R.id.pano_shutter_button); mShutterButton.setOnClickListener(new View.OnClickListener() { @Override @@ -343,7 +349,6 @@ public class PanoramaActivity extends Activity implements mCaptureLayout.setVisibility(View.INVISIBLE); mPreview.setVisibility(View.INVISIBLE); mReviewLayout.setVisibility(View.VISIBLE); - mCaptureView.setBitmap(null); mCaptureView.setStatusText(""); mCaptureView.setSweepAngle(0); } @@ -354,6 +359,7 @@ public class PanoramaActivity extends Activity implements releaseCamera(); mMosaicFrameProcessor.onPause(); mCaptureView.onPause(); + mRealTimeMosaicView.onPause(); mSensorManager.unregisterListener(mListener); System.gc(); } @@ -383,6 +389,7 @@ public class PanoramaActivity extends Activity implements mMosaicFrameProcessor.onResume(); } mCaptureView.onResume(); + mRealTimeMosaicView.onResume(); } private final SensorEventListener mListener = new SensorEventListener() { |