From 595644d0f542cad71520005da0dadb1ef371ba9e Mon Sep 17 00:00:00 2001 From: Wei Jia Date: Wed, 3 Jun 2015 13:47:51 -0700 Subject: libstagefright: check memory size for overflow before allocation. Bug: 20674086 Change-Id: I431aa2b7d30a942350ab6d105451c6b77e2f99d4 (cherry picked from commit 42cccd7c8811597d56fb86afeacf6231d693dea6) Tested-by: Wolfgang Wiedmeyer --- .../codecs/m4v_h263/dec/src/pvdec_api.cpp | 39 ++++++++++++++++++++++ .../codecs/m4v_h263/enc/src/mp4enc_api.cpp | 26 +++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp b/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp index 0802e29..b1a39e6 100644 --- a/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp +++ b/media/libstagefright/codecs/m4v_h263/dec/src/pvdec_api.cpp @@ -106,6 +106,11 @@ OSCL_EXPORT_REF Bool PVInitVideoDecoder(VideoDecControls *decCtrl, uint8 *volbuf #ifdef DEC_INTERNAL_MEMORY_OPT video->vol = (Vol **) IMEM_VOL; #else + if ((size_t)nLayers > SIZE_MAX / sizeof(Vol *)) { + status = PV_FALSE; + goto fail; + } + video->vol = (Vol **) oscl_malloc(nLayers * sizeof(Vol *)); #endif if (video->vol == NULL) status = PV_FALSE; @@ -139,6 +144,11 @@ OSCL_EXPORT_REF Bool PVInitVideoDecoder(VideoDecControls *decCtrl, uint8 *volbuf else oscl_memset(video->prevVop, 0, sizeof(Vop)); video->memoryUsage += (sizeof(Vop) * 2); + if ((size_t)nLayers > SIZE_MAX / sizeof(Vop *)) { + status = PV_FALSE; + goto fail; + } + video->vopHeader = (Vop **) oscl_malloc(sizeof(Vop *) * nLayers); #endif if (video->vopHeader == NULL) status = PV_FALSE; @@ -285,6 +295,7 @@ OSCL_EXPORT_REF Bool PVInitVideoDecoder(VideoDecControls *decCtrl, uint8 *volbuf status = PV_FALSE; } +fail: if (status == PV_FALSE) PVCleanUpVideoDecoder(decCtrl); return status; @@ -311,6 +322,10 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay video->nMBPerRow * video->nMBPerCol; } + if (((uint64_t)video->width * video->height) > (uint64_t)INT32_MAX / sizeof(PIXEL)) { + return PV_FALSE; + } + size = (int32)sizeof(PIXEL) * video->width * video->height; #ifdef PV_MEMORY_POOL decCtrl->size = size; @@ -326,6 +341,9 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay video->prevVop->uChan = video->prevVop->yChan + size; video->prevVop->vChan = video->prevVop->uChan + (size >> 2); #else + if (size > INT32_MAX / 3 * 2) { + return PV_FALSE; + } video->currVop->yChan = (PIXEL *) oscl_malloc(size * 3 / 2); /* Allocate memory for all VOP OKA 3/2/1*/ if (video->currVop->yChan == NULL) status = PV_FALSE; @@ -353,6 +371,10 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay { oscl_memset(video->prevEnhcVop, 0, sizeof(Vop)); #ifndef PV_MEMORY_POOL + if (size > INT32_MAX / 3 * 2) { + return PV_FALSE; + } + video->prevEnhcVop->yChan = (PIXEL *) oscl_malloc(size * 3 / 2); /* Allocate memory for all VOP OKA 3/2/1*/ if (video->prevEnhcVop->yChan == NULL) status = PV_FALSE; video->prevEnhcVop->uChan = video->prevEnhcVop->yChan + size; @@ -409,10 +431,17 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay if (video->acPredFlag == NULL) status = PV_FALSE; video->memoryUsage += (nTotalMB); + if ((size_t)nTotalMB > SIZE_MAX / sizeof(typeDCStore)) { + return PV_FALSE; + } video->predDC = (typeDCStore *) oscl_malloc(nTotalMB * sizeof(typeDCStore)); if (video->predDC == NULL) status = PV_FALSE; video->memoryUsage += (nTotalMB * sizeof(typeDCStore)); + if (nMBPerRow > INT32_MAX - 1 + || (size_t)(nMBPerRow + 1) > SIZE_MAX / sizeof(typeDCACStore)) { + return PV_FALSE; + } video->predDCAC_col = (typeDCACStore *) oscl_malloc((nMBPerRow + 1) * sizeof(typeDCACStore)); if (video->predDCAC_col == NULL) status = PV_FALSE; video->memoryUsage += ((nMBPerRow + 1) * sizeof(typeDCACStore)); @@ -428,6 +457,10 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay video->headerInfo.CBP = (uint8 *) oscl_malloc(nTotalMB); if (video->headerInfo.CBP == NULL) status = PV_FALSE; video->memoryUsage += nTotalMB; + + if ((size_t)nTotalMB > SIZE_MAX / sizeof(int16)) { + return PV_FALSE; + } video->QPMB = (int16 *) oscl_malloc(nTotalMB * sizeof(int16)); if (video->QPMB == NULL) status = PV_FALSE; video->memoryUsage += (nTotalMB * sizeof(int)); @@ -445,6 +478,9 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay video->memoryUsage += sizeof(MacroBlock); } /* Allocating motion vector space */ + if ((size_t)nTotalMB > SIZE_MAX / (sizeof(MOT) * 4)) { + return PV_FALSE; + } video->motX = (MOT *) oscl_malloc(sizeof(MOT) * 4 * nTotalMB); if (video->motX == NULL) status = PV_FALSE; video->motY = (MOT *) oscl_malloc(sizeof(MOT) * 4 * nTotalMB); @@ -478,6 +514,9 @@ Bool PVAllocVideoData(VideoDecControls *decCtrl, int width, int height, int nLay } #else + if (nTotalMB > INT32_MAX / 6) { + return PV_FALSE; + } video->pstprcTypCur = (uint8 *) oscl_malloc(nTotalMB * 6); video->memoryUsage += (nTotalMB * 6); if (video->pstprcTypCur == NULL) diff --git a/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp b/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp index 5ea0de4..6c4510e 100644 --- a/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp +++ b/media/libstagefright/codecs/m4v_h263/enc/src/mp4enc_api.cpp @@ -619,6 +619,10 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid max = temp_w * temp_h; max_width = ((temp_w + 15) >> 4) << 4; max_height = ((temp_h + 15) >> 4) << 4; + if (((uint64_t)max_width * max_height) > (uint64_t)INT32_MAX + || temp_w > INT32_MAX - 15 || temp_h > INT32_MAX - 15) { + goto CLEAN_UP; + } nTotalMB = ((max_width * max_height) >> 8); } @@ -663,6 +667,9 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid /* Allocating motion vector space and interpolation memory*/ + if ((size_t)nTotalMB > SIZE_MAX / sizeof(MOT *)) { + goto CLEAN_UP; + } video->mot = (MOT **)M4VENC_MALLOC(sizeof(MOT *) * nTotalMB); if (video->mot == NULL) goto CLEAN_UP; @@ -685,11 +692,17 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid /* so that compilers can generate faster code to indexing the */ /* data inside (by using << instead of *). 04/14/2000. */ /* 5/29/01, use decoder lib ACDC prediction memory scheme. */ + if ((size_t)nTotalMB > SIZE_MAX / sizeof(typeDCStore)) { + goto CLEAN_UP; + } video->predDC = (typeDCStore *) M4VENC_MALLOC(nTotalMB * sizeof(typeDCStore)); if (video->predDC == NULL) goto CLEAN_UP; if (!video->encParams->H263_Enabled) { + if ((size_t)((max_width >> 4) + 1) > SIZE_MAX / sizeof(typeDCACStore)) { + goto CLEAN_UP; + } video->predDCAC_col = (typeDCACStore *) M4VENC_MALLOC(((max_width >> 4) + 1) * sizeof(typeDCACStore)); if (video->predDCAC_col == NULL) goto CLEAN_UP; @@ -697,6 +710,9 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid /* the rest will be used for storing horizontal (row) AC coefficients */ video->predDCAC_row = video->predDCAC_col + 1; /* ACDC */ + if ((size_t)nTotalMB > SIZE_MAX / sizeof(Int)) { + goto CLEAN_UP; + } video->acPredFlag = (Int *) M4VENC_MALLOC(nTotalMB * sizeof(Int)); /* Memory for acPredFlag */ if (video->acPredFlag == NULL) goto CLEAN_UP; } @@ -750,8 +766,15 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid offset = (pitch << 4) + 16; max_height += 32; } + if (((uint64_t)pitch * max_height) > (uint64_t)INT32_MAX) { + goto CLEAN_UP; + } size = pitch * max_height; + if (size > INT32_MAX - (size >> 1) + || (size_t)(size + (size >> 1)) > SIZE_MAX / sizeof(PIXEL)) { + goto CLEAN_UP; + } video->currVop->yChan = (PIXEL *)M4VENC_MALLOC(sizeof(PIXEL) * (size + (size >> 1))); /* Memory for currVop Y */ if (video->currVop->yChan == NULL) goto CLEAN_UP; video->currVop->uChan = video->currVop->yChan + size;/* Memory for currVop U */ @@ -850,6 +873,9 @@ OSCL_EXPORT_REF Bool PVInitVideoEncoder(VideoEncControls *encoderControl, Vid /* /// End /////////////////////////////////////// */ + if ((size_t)nLayers > SIZE_MAX / sizeof(Vol *)) { + goto CLEAN_UP; + } video->vol = (Vol **)M4VENC_MALLOC(nLayers * sizeof(Vol *)); /* Memory for VOL pointers */ /* Memory allocation and Initialization of Vols and writing of headers */ -- cgit v1.1