libstagefright: check memory size for overflow before allocation.

Bug: 20674086
Change-Id: I431aa2b7d30a942350ab6d105451c6b77e2f99d4
(cherry picked from commit 42cccd7c8811597d56fb86afeacf6231d693dea6)
Tested-by: Wolfgang Wiedmeyer <wolfgit@wiedmeyer.de>

2 files changed, 65 insertions, 0 deletions

diff --git a/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp b/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp
index 0802e29..b1a39e6 100644
--- a/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp
+++ b/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp
@@ -106,6 +106,11 @@ OSCL_EXPORT_REF Bool PVInitVideoDecoder(VideoDecControls *decCtrl, uint8 *volbuf
 #ifdef DEC_INTERNAL_MEMORY_OPT
         video->vol = (Vol **) IMEM_VOL;
 #else
+        if ((size_t)nLayers > SIZE_MAX / sizeof(Vol *)) {
+            status = PV_FALSE;
+            goto fail;
+        }
+
         video->vol = (Vol **) oscl_malloc(nLayers * sizeof(Vol *));
 #endif
         if (video->vol == NULL) status = PV_FALSE;
@@ -139,6 +144,11 @@ OSCL_EXPORT_REF Bool PVInitVideoDecoder(VideoDecControls *decCtrl, uint8 *volbuf
         else oscl_memset(video->prevVop, 0, sizeof(Vop));
         video->memoryUsage += (sizeof(Vop) * 2);
 
+        if ((size_t)nLayers > SIZE_MAX / sizeof(Vop *)) {
+            status = PV_FALSE;
+            goto fail;
+        }
+
         video->vopHeader = (Vop **) oscl_malloc(sizeof(Vop *) * nLayers);
 #endif
         if (video->vopHeader == NULL) status = PV_FALSE;
@@ -285,6 +295,7 @@ OSCL_EXPORT_REF Bool PVInitVideoDecoder(VideoDecControls *decCtrl, uint8 *volbuf
         status = PV_FALSE;
     }
 
+fail:
     if (status == PV_FALSE) PVCleanUpVideoDecoder(decCtrl);
 
     return status;
@@ -311,6 +322,10 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
             video->nMBPerRow * video->nMBPerCol;
     }
 
+    if (((uint64_t)video->width * video->height) > (uint64_t)INT32_MAX / sizeof(PIXEL)) {
+        return PV_FALSE;
+    }
+
     size = (int32)sizeof(PIXEL) * video->width * video->height;
 #ifdef PV_MEMORY_POOL
     decCtrl->size = size;
@@ -326,6 +341,9 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
     video->prevVop->uChan = video->prevVop->yChan + size;
     video->prevVop->vChan = video->prevVop->uChan + (size >> 2);
 #else
+    if (size > INT32_MAX / 3 * 2) {
+        return PV_FALSE;
+    }
     video->currVop->yChan = (PIXEL *) oscl_malloc(size * 3 / 2); /* Allocate memory for all VOP OKA 3/2/1*/
     if (video->currVop->yChan == NULL) status = PV_FALSE;
 
@@ -353,6 +371,10 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
         {
             oscl_memset(video->prevEnhcVop, 0, sizeof(Vop));
 #ifndef PV_MEMORY_POOL
+            if (size > INT32_MAX / 3 * 2) {
+                return PV_FALSE;
+            }
+
             video->prevEnhcVop->yChan = (PIXEL *) oscl_malloc(size * 3 / 2); /* Allocate memory for all VOP OKA 3/2/1*/
             if (video->prevEnhcVop->yChan == NULL) status = PV_FALSE;
             video->prevEnhcVop->uChan = video->prevEnhcVop->yChan + size;
@@ -409,10 +431,17 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
     if (video->acPredFlag == NULL) status = PV_FALSE;
     video->memoryUsage += (nTotalMB);
 
+    if ((size_t)nTotalMB > SIZE_MAX / sizeof(typeDCStore)) {
+        return PV_FALSE;
+    }
     video->predDC = (typeDCStore *) oscl_malloc(nTotalMB * sizeof(typeDCStore));
     if (video->predDC == NULL) status = PV_FALSE;
     video->memoryUsage += (nTotalMB * sizeof(typeDCStore));
 
+    if (nMBPerRow > INT32_MAX - 1
+            || (size_t)(nMBPerRow + 1) > SIZE_MAX / sizeof(typeDCACStore)) {
+        return PV_FALSE;
+    }
     video->predDCAC_col = (typeDCACStore *) oscl_malloc((nMBPerRow + 1) * sizeof(typeDCACStore));
     if (video->predDCAC_col == NULL) status = PV_FALSE;
     video->memoryUsage += ((nMBPerRow + 1) * sizeof(typeDCACStore));
@@ -428,6 +457,10 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
     video->headerInfo.CBP = (uint8 *) oscl_malloc(nTotalMB);
     if (video->headerInfo.CBP == NULL) status = PV_FALSE;
     video->memoryUsage += nTotalMB;
+
+    if ((size_t)nTotalMB > SIZE_MAX / sizeof(int16)) {
+        return PV_FALSE;
+    }
     video->QPMB = (int16 *) oscl_malloc(nTotalMB * sizeof(int16));
     if (video->QPMB == NULL) status = PV_FALSE;
     video->memoryUsage += (nTotalMB * sizeof(int));
@@ -445,6 +478,9 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
         video->memoryUsage += sizeof(MacroBlock);
     }
     /* Allocating motion vector space */
+    if ((size_t)nTotalMB > SIZE_MAX / (sizeof(MOT) * 4)) {
+        return PV_FALSE;
+    }
     video->motX = (MOT *) oscl_malloc(sizeof(MOT) * 4 * nTotalMB);
     if (video->motX == NULL) status = PV_FALSE;
     video->motY = (MOT *) oscl_malloc(sizeof(MOT) * 4 * nTotalMB);
@@ -478,6 +514,9 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay
     }
 
 #else
+    if (nTotalMB > INT32_MAX / 6) {
+        return PV_FALSE;
+    }
     video->pstprcTypCur = (uint8 *) oscl_malloc(nTotalMB * 6);
     video->memoryUsage += (nTotalMB * 6);
     if (video->pstprcTypCur == NULL)
diff --git a/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp b/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp
index 5ea0de4..6c4510e 100644
--- a/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp
+++ b/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp
@@ -619,6 +619,10 @@ OSCL_EXPORT_REF Bool    PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
             max = temp_w * temp_h;
             max_width = ((temp_w + 15) >> 4) << 4;
             max_height = ((temp_h + 15) >> 4) << 4;
+            if (((uint64_t)max_width * max_height) > (uint64_t)INT32_MAX
+                    || temp_w > INT32_MAX - 15 || temp_h > INT32_MAX - 15) {
+                goto CLEAN_UP;
+            }
             nTotalMB = ((max_width * max_height) >> 8);
         }
 
@@ -663,6 +667,9 @@ OSCL_EXPORT_REF Bool    PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
 
     /* Allocating motion vector space and interpolation memory*/
 
+    if ((size_t)nTotalMB > SIZE_MAX / sizeof(MOT *)) {
+        goto CLEAN_UP;
+    }
     video->mot = (MOT **)M4VENC_MALLOC(sizeof(MOT *) * nTotalMB);
     if (video->mot == NULL) goto CLEAN_UP;
 
@@ -685,11 +692,17 @@ OSCL_EXPORT_REF Bool    PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
     /*    so that compilers can generate faster code to indexing the     */
     /*    data inside (by using << instead of *).         04/14/2000. */
     /* 5/29/01, use  decoder lib ACDC prediction memory scheme.  */
+    if ((size_t)nTotalMB > SIZE_MAX / sizeof(typeDCStore)) {
+        goto CLEAN_UP;
+    }
     video->predDC = (typeDCStore *) M4VENC_MALLOC(nTotalMB * sizeof(typeDCStore));
     if (video->predDC == NULL) goto CLEAN_UP;
 
     if (!video->encParams->H263_Enabled)
     {
+        if ((size_t)((max_width >> 4) + 1) > SIZE_MAX / sizeof(typeDCACStore)) {
+            goto CLEAN_UP;
+        }
         video->predDCAC_col = (typeDCACStore *) M4VENC_MALLOC(((max_width >> 4) + 1) * sizeof(typeDCACStore));
         if (video->predDCAC_col == NULL) goto CLEAN_UP;
 
@@ -697,6 +710,9 @@ OSCL_EXPORT_REF Bool    PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
         /*  the rest will be used for storing horizontal (row) AC coefficients  */
         video->predDCAC_row = video->predDCAC_col + 1;        /*  ACDC */
 
+        if ((size_t)nTotalMB > SIZE_MAX / sizeof(Int)) {
+            goto CLEAN_UP;
+        }
         video->acPredFlag = (Int *) M4VENC_MALLOC(nTotalMB * sizeof(Int)); /* Memory for acPredFlag */
         if (video->acPredFlag == NULL) goto CLEAN_UP;
     }
@@ -750,8 +766,15 @@ OSCL_EXPORT_REF Bool    PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
         offset = (pitch << 4) + 16;
         max_height += 32;
     }
+    if (((uint64_t)pitch * max_height) > (uint64_t)INT32_MAX) {
+        goto CLEAN_UP;
+    }
     size = pitch * max_height;
 
+    if (size > INT32_MAX - (size >> 1)
+            || (size_t)(size + (size >> 1)) > SIZE_MAX / sizeof(PIXEL)) {
+        goto CLEAN_UP;
+    }
     video->currVop->yChan = (PIXEL *)M4VENC_MALLOC(sizeof(PIXEL) * (size + (size >> 1))); /* Memory for currVop Y */
     if (video->currVop->yChan == NULL) goto CLEAN_UP;
     video->currVop->uChan = video->currVop->yChan + size;/* Memory for currVop U */
@@ -850,6 +873,9 @@ OSCL_EXPORT_REF Bool    PVInitVideoEncoder(VideoEncControls *encoderControl, Vid
     /* /// End /////////////////////////////////////// */
 
 
+    if ((size_t)nLayers > SIZE_MAX / sizeof(Vol *)) {
+        goto CLEAN_UP;
+    }
     video->vol = (Vol **)M4VENC_MALLOC(nLayers * sizeof(Vol *)); /* Memory for VOL pointers */
 
     /* Memory allocation and Initialization of Vols and writing of headers */