diff options
Diffstat (limited to 'third_party/sqlite/ext/fts3/fts3.c')
| -rw-r--r-- | third_party/sqlite/ext/fts3/fts3.c | 927 |
1 files changed, 658 insertions, 269 deletions
diff --git a/third_party/sqlite/ext/fts3/fts3.c b/third_party/sqlite/ext/fts3/fts3.c index a17cf38..d75058c 100644 --- a/third_party/sqlite/ext/fts3/fts3.c +++ b/third_party/sqlite/ext/fts3/fts3.c @@ -271,6 +271,7 @@ ** deletions and duplications. This would basically be a forced merge ** into a single segment. */ +#define CHROMIUM_FTS3_CHANGES 1 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) @@ -313,6 +314,16 @@ # define FTSTRACE(A) #endif +#if 0 +/* Useful to set breakpoints. See main.c sqlite3Corrupt(). */ +static int fts3Corrupt(void){ + return SQLITE_CORRUPT; +} +# define SQLITE_CORRUPT_BKPT fts3Corrupt() +#else +# define SQLITE_CORRUPT_BKPT SQLITE_CORRUPT +#endif + /* It is not safe to call isspace(), tolower(), or isalnum() on ** hi-bit-set characters. This is the same solution used in the ** tokenizer. @@ -401,30 +412,41 @@ static int fts3PutVarint(char *p, sqlite_int64 v){ /* Read a 64-bit variable-length integer from memory starting at p[0]. * Return the number of bytes read, or 0 on error. * The value is stored in *v. */ -static int fts3GetVarint(const char *p, sqlite_int64 *v){ +static int fts3GetVarintSafe(const char *p, sqlite_int64 *v, int max){ const unsigned char *q = (const unsigned char *) p; sqlite_uint64 x = 0, y = 1; - while( (*q & 0x80) == 0x80 ){ + if( max>VARINT_MAX ) max = VARINT_MAX; + while( max && (*q & 0x80) == 0x80 ){ + max--; x += y * (*q++ & 0x7f); y <<= 7; - if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */ - assert( 0 ); - return 0; - } + } + if( !max ){ + assert( 0 ); + return 0; /* tried to read too much; bad data */ } x += y * (*q++); *v = (sqlite_int64) x; return (int) (q - (unsigned char *)p); } -static int fts3GetVarint32(const char *p, int *pi){ +static int fts3GetVarint(const char *p, sqlite_int64 *v){ + return fts3GetVarintSafe(p, v, VARINT_MAX); +} + +static int fts3GetVarint32Safe(const char *p, int *pi, int max){ sqlite_int64 i; - int ret = fts3GetVarint(p, &i); + int ret = fts3GetVarintSafe(p, &i, max); + if( !ret ) return ret; *pi = (int) i; assert( *pi==i ); return ret; } +static int fts3GetVarint32(const char* p, int *pi){ + return fts3GetVarint32Safe(p, pi, VARINT_MAX); +} + /*******************************************************************/ /* DataBuffer is used to collect data into a buffer in piecemeal ** fashion. It implements the usual distinction between amount of @@ -593,7 +615,7 @@ typedef struct DLReader { static int dlrAtEnd(DLReader *pReader){ assert( pReader->nData>=0 ); - return pReader->nData==0; + return pReader->nData<=0; } static sqlite_int64 dlrDocid(DLReader *pReader){ assert( !dlrAtEnd(pReader) ); @@ -617,7 +639,8 @@ static int dlrAllDataBytes(DLReader *pReader){ */ static const char *dlrPosData(DLReader *pReader){ sqlite_int64 iDummy; - int n = fts3GetVarint(pReader->pData, &iDummy); + int n = fts3GetVarintSafe(pReader->pData, &iDummy, pReader->nElement); + if( !n ) return NULL; assert( !dlrAtEnd(pReader) ); return pReader->pData+n; } @@ -627,7 +650,7 @@ static int dlrPosDataLen(DLReader *pReader){ assert( !dlrAtEnd(pReader) ); return pReader->nElement-n; } -static void dlrStep(DLReader *pReader){ +static int dlrStep(DLReader *pReader){ assert( !dlrAtEnd(pReader) ); /* Skip past current doclist element. */ @@ -636,32 +659,44 @@ static void dlrStep(DLReader *pReader){ pReader->nData -= pReader->nElement; /* If there is more data, read the next doclist element. */ - if( pReader->nData!=0 ){ + if( pReader->nData>0 ){ sqlite_int64 iDocidDelta; - int iDummy, n = fts3GetVarint(pReader->pData, &iDocidDelta); + int nTotal = 0; + int iDummy, n = fts3GetVarintSafe(pReader->pData, &iDocidDelta, pReader->nData); + if( !n ) return SQLITE_CORRUPT_BKPT; + nTotal += n; pReader->iDocid += iDocidDelta; if( pReader->iType>=DL_POSITIONS ){ - assert( n<pReader->nData ); while( 1 ){ - n += fts3GetVarint32(pReader->pData+n, &iDummy); - assert( n<=pReader->nData ); + n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal); + if( !n ) return SQLITE_CORRUPT_BKPT; + nTotal += n; if( iDummy==POS_END ) break; if( iDummy==POS_COLUMN ){ - n += fts3GetVarint32(pReader->pData+n, &iDummy); - assert( n<pReader->nData ); + n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal); + if( !n ) return SQLITE_CORRUPT_BKPT; + nTotal += n; }else if( pReader->iType==DL_POSITIONS_OFFSETS ){ - n += fts3GetVarint32(pReader->pData+n, &iDummy); - n += fts3GetVarint32(pReader->pData+n, &iDummy); - assert( n<pReader->nData ); + n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal); + if( !n ) return SQLITE_CORRUPT_BKPT; + nTotal += n; + n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal); + if( !n ) return SQLITE_CORRUPT_BKPT; + nTotal += n; } } } - pReader->nElement = n; + pReader->nElement = nTotal; assert( pReader->nElement<=pReader->nData ); } + return SQLITE_OK; } -static void dlrInit(DLReader *pReader, DocListType iType, - const char *pData, int nData){ +static void dlrDestroy(DLReader *pReader){ + SCRAMBLE(pReader); +} +static int dlrInit(DLReader *pReader, DocListType iType, + const char *pData, int nData){ + int rc; assert( pData!=NULL && nData!=0 ); pReader->iType = iType; pReader->pData = pData; @@ -670,10 +705,9 @@ static void dlrInit(DLReader *pReader, DocListType iType, pReader->iDocid = 0; /* Load the first element's data. There must be a first element. */ - dlrStep(pReader); -} -static void dlrDestroy(DLReader *pReader){ - SCRAMBLE(pReader); + rc = dlrStep(pReader); + if( rc!=SQLITE_OK ) dlrDestroy(pReader); + return rc; } #ifndef NDEBUG @@ -760,9 +794,9 @@ static void dlwDestroy(DLWriter *pWriter){ /* TODO(shess) This has become just a helper for docListMerge. ** Consider a refactor to make this cleaner. */ -static void dlwAppend(DLWriter *pWriter, - const char *pData, int nData, - sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){ +static int dlwAppend(DLWriter *pWriter, + const char *pData, int nData, + sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){ sqlite_int64 iDocid = 0; char c[VARINT_MAX]; int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */ @@ -771,7 +805,8 @@ static void dlwAppend(DLWriter *pWriter, #endif /* Recode the initial docid as delta from iPrevDocid. */ - nFirstOld = fts3GetVarint(pData, &iDocid); + nFirstOld = fts3GetVarintSafe(pData, &iDocid, nData); + if( !nFirstOld ) return SQLITE_CORRUPT_BKPT; assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) ); nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid); @@ -792,10 +827,11 @@ static void dlwAppend(DLWriter *pWriter, dataBufferAppend(pWriter->b, c, nFirstNew); } pWriter->iPrevDocid = iLastDocid; + return SQLITE_OK; } -static void dlwCopy(DLWriter *pWriter, DLReader *pReader){ - dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader), - dlrDocid(pReader), dlrDocid(pReader)); +static int dlwCopy(DLWriter *pWriter, DLReader *pReader){ + return dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader), + dlrDocid(pReader), dlrDocid(pReader)); } static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){ char c[VARINT_MAX]; @@ -856,45 +892,61 @@ static int plrEndOffset(PLReader *pReader){ assert( !plrAtEnd(pReader) ); return pReader->iEndOffset; } -static void plrStep(PLReader *pReader){ - int i, n; +static int plrStep(PLReader *pReader){ + int i, n, nTotal = 0; assert( !plrAtEnd(pReader) ); - if( pReader->nData==0 ){ + if( pReader->nData<=0 ){ pReader->pData = NULL; - return; + return SQLITE_OK; } - n = fts3GetVarint32(pReader->pData, &i); + n = fts3GetVarint32Safe(pReader->pData, &i, pReader->nData); + if( !n ) return SQLITE_CORRUPT_BKPT; + nTotal += n; if( i==POS_COLUMN ){ - n += fts3GetVarint32(pReader->pData+n, &pReader->iColumn); + n = fts3GetVarint32Safe(pReader->pData+nTotal, &pReader->iColumn, pReader->nData-nTotal); + if( !n ) return SQLITE_CORRUPT_BKPT; + nTotal += n; pReader->iPosition = 0; pReader->iStartOffset = 0; - n += fts3GetVarint32(pReader->pData+n, &i); + n = fts3GetVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal); + if( !n ) return SQLITE_CORRUPT_BKPT; + nTotal += n; } /* Should never see adjacent column changes. */ assert( i!=POS_COLUMN ); if( i==POS_END ){ + assert( nTotal<=pReader->nData ); pReader->nData = 0; pReader->pData = NULL; - return; + return SQLITE_OK; } pReader->iPosition += i-POS_BASE; if( pReader->iType==DL_POSITIONS_OFFSETS ){ - n += fts3GetVarint32(pReader->pData+n, &i); + n = fts3GetVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal); + if( !n ) return SQLITE_CORRUPT_BKPT; + nTotal += n; pReader->iStartOffset += i; - n += fts3GetVarint32(pReader->pData+n, &i); + n = fts3GetVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal); + if( !n ) return SQLITE_CORRUPT_BKPT; + nTotal += n; pReader->iEndOffset = pReader->iStartOffset+i; } - assert( n<=pReader->nData ); - pReader->pData += n; - pReader->nData -= n; + assert( nTotal<=pReader->nData ); + pReader->pData += nTotal; + pReader->nData -= nTotal; + return SQLITE_OK; } -static void plrInit(PLReader *pReader, DLReader *pDLReader){ +static void plrDestroy(PLReader *pReader){ + SCRAMBLE(pReader); +} +static int plrInit(PLReader *pReader, DLReader *pDLReader){ + int rc; pReader->pData = dlrPosData(pDLReader); pReader->nData = dlrPosDataLen(pDLReader); pReader->iType = pDLReader->iType; @@ -902,10 +954,9 @@ static void plrInit(PLReader *pReader, DLReader *pDLReader){ pReader->iPosition = 0; pReader->iStartOffset = 0; pReader->iEndOffset = 0; - plrStep(pReader); -} -static void plrDestroy(PLReader *pReader){ - SCRAMBLE(pReader); + rc = plrStep(pReader); + if( rc!=SQLITE_OK ) plrDestroy(pReader); + return rc; } /*******************************************************************/ @@ -1091,14 +1142,16 @@ static void dlcDelete(DLCollector *pCollector){ ** deletion will be trimmed, and will thus not effect a deletion ** during the merge. */ -static void docListTrim(DocListType iType, const char *pData, int nData, - int iColumn, DocListType iOutType, DataBuffer *out){ +static int docListTrim(DocListType iType, const char *pData, int nData, + int iColumn, DocListType iOutType, DataBuffer *out){ DLReader dlReader; DLWriter dlWriter; + int rc; assert( iOutType<=iType ); - dlrInit(&dlReader, iType, pData, nData); + rc = dlrInit(&dlReader, iType, pData, nData); + if( rc!=SQLITE_OK ) return rc; dlwInit(&dlWriter, iOutType, out); while( !dlrAtEnd(&dlReader) ){ @@ -1106,7 +1159,8 @@ static void docListTrim(DocListType iType, const char *pData, int nData, PLWriter plWriter; int match = 0; - plrInit(&plReader, &dlReader); + rc = plrInit(&plReader, &dlReader); + if( rc!=SQLITE_OK ) break; while( !plrAtEnd(&plReader) ){ if( iColumn==-1 || plrColumn(&plReader)==iColumn ){ @@ -1117,7 +1171,11 @@ static void docListTrim(DocListType iType, const char *pData, int nData, plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader), plrStartOffset(&plReader), plrEndOffset(&plReader)); } - plrStep(&plReader); + rc = plrStep(&plReader); + if( rc!=SQLITE_OK ){ + plrDestroy(&plReader); + goto err; + } } if( match ){ plwTerminate(&plWriter); @@ -1125,10 +1183,13 @@ static void docListTrim(DocListType iType, const char *pData, int nData, } plrDestroy(&plReader); - dlrStep(&dlReader); + rc = dlrStep(&dlReader); + if( rc!=SQLITE_OK ) break; } +err: dlwDestroy(&dlWriter); dlrDestroy(&dlReader); + return rc; } /* Used by docListMerge() to keep doclists in the ascending order by @@ -1185,19 +1246,20 @@ static void orderedDLReaderReorder(OrderedDLReader *p, int n){ /* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably ** be fixed. */ -static void docListMerge(DataBuffer *out, - DLReader *pReaders, int nReaders){ +static int docListMerge(DataBuffer *out, + DLReader *pReaders, int nReaders){ OrderedDLReader readers[MERGE_COUNT]; DLWriter writer; int i, n; const char *pStart = 0; int nStart = 0; sqlite_int64 iFirstDocid = 0, iLastDocid = 0; + int rc = SQLITE_OK; assert( nReaders>0 ); if( nReaders==1 ){ dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders)); - return; + return SQLITE_OK; } assert( nReaders<=MERGE_COUNT ); @@ -1230,20 +1292,23 @@ static void docListMerge(DataBuffer *out, nStart += dlrDocDataBytes(readers[0].pReader); }else{ if( pStart!=0 ){ - dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); + rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); + if( rc!=SQLITE_OK ) goto err; } pStart = dlrDocData(readers[0].pReader); nStart = dlrDocDataBytes(readers[0].pReader); iFirstDocid = iDocid; } iLastDocid = iDocid; - dlrStep(readers[0].pReader); + rc = dlrStep(readers[0].pReader); + if( rc!= SQLITE_OK ) goto err; /* Drop all of the older elements with the same docid. */ for(i=1; i<nReaders && !dlrAtEnd(readers[i].pReader) && dlrDocid(readers[i].pReader)==iDocid; i++){ - dlrStep(readers[i].pReader); + rc = dlrStep(readers[i].pReader); + if( rc!=SQLITE_OK ) goto err; } /* Get the readers back into order. */ @@ -1253,8 +1318,10 @@ static void docListMerge(DataBuffer *out, } /* Copy over any remaining elements. */ - if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); + if( nStart>0 ) rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); +err: dlwDestroy(&writer); + return rc; } /* Helper function for posListUnion(). Compares the current position @@ -1290,30 +1357,40 @@ static int posListCmp(PLReader *pLeft, PLReader *pRight){ ** work with any doclist type, though both inputs and the output ** should be the same type. */ -static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){ +static int posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){ PLReader left, right; PLWriter writer; + int rc; assert( dlrDocid(pLeft)==dlrDocid(pRight) ); assert( pLeft->iType==pRight->iType ); assert( pLeft->iType==pOut->iType ); - plrInit(&left, pLeft); - plrInit(&right, pRight); + rc = plrInit(&left, pLeft); + if( rc!=SQLITE_OK ) return rc; + rc = plrInit(&right, pRight); + if( rc!=SQLITE_OK ){ + plrDestroy(&left); + return rc; + } plwInit(&writer, pOut, dlrDocid(pLeft)); while( !plrAtEnd(&left) || !plrAtEnd(&right) ){ int c = posListCmp(&left, &right); if( c<0 ){ plwCopy(&writer, &left); - plrStep(&left); + rc = plrStep(&left); + if( rc!=SQLITE_OK ) break; }else if( c>0 ){ plwCopy(&writer, &right); - plrStep(&right); + rc = plrStep(&right); + if( rc!=SQLITE_OK ) break; }else{ plwCopy(&writer, &left); - plrStep(&left); - plrStep(&right); + rc = plrStep(&left); + if( rc!=SQLITE_OK ) break; + rc = plrStep(&right); + if( rc!=SQLITE_OK ) break; } } @@ -1321,56 +1398,75 @@ static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){ plwDestroy(&writer); plrDestroy(&left); plrDestroy(&right); + return rc; } /* Write the union of doclists in pLeft and pRight to pOut. For ** docids in common between the inputs, the union of the position ** lists is written. Inputs and outputs are always type DL_DEFAULT. */ -static void docListUnion( +static int docListUnion( const char *pLeft, int nLeft, const char *pRight, int nRight, DataBuffer *pOut /* Write the combined doclist here */ ){ DLReader left, right; DLWriter writer; + int rc; if( nLeft==0 ){ if( nRight!=0) dataBufferAppend(pOut, pRight, nRight); - return; + return SQLITE_OK; } if( nRight==0 ){ dataBufferAppend(pOut, pLeft, nLeft); - return; + return SQLITE_OK; } - dlrInit(&left, DL_DEFAULT, pLeft, nLeft); - dlrInit(&right, DL_DEFAULT, pRight, nRight); + rc = dlrInit(&left, DL_DEFAULT, pLeft, nLeft); + if( rc!=SQLITE_OK ) return rc; + rc = dlrInit(&right, DL_DEFAULT, pRight, nRight); + if( rc!=SQLITE_OK){ + dlrDestroy(&left); + return rc; + } dlwInit(&writer, DL_DEFAULT, pOut); while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){ if( dlrAtEnd(&right) ){ - dlwCopy(&writer, &left); - dlrStep(&left); + rc = dlwCopy(&writer, &left); + if( rc!=SQLITE_OK) break; + rc = dlrStep(&left); + if( rc!=SQLITE_OK) break; }else if( dlrAtEnd(&left) ){ - dlwCopy(&writer, &right); - dlrStep(&right); + rc = dlwCopy(&writer, &right); + if( rc!=SQLITE_OK ) break; + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) break; }else if( dlrDocid(&left)<dlrDocid(&right) ){ - dlwCopy(&writer, &left); - dlrStep(&left); + rc = dlwCopy(&writer, &left); + if( rc!=SQLITE_OK ) break; + rc = dlrStep(&left); + if( rc!=SQLITE_OK ) break; }else if( dlrDocid(&left)>dlrDocid(&right) ){ - dlwCopy(&writer, &right); - dlrStep(&right); + rc = dlwCopy(&writer, &right); + if( rc!=SQLITE_OK ) break; + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) break; }else{ - posListUnion(&left, &right, &writer); - dlrStep(&left); - dlrStep(&right); + rc = posListUnion(&left, &right, &writer); + if( rc!=SQLITE_OK ) break; + rc = dlrStep(&left); + if( rc!=SQLITE_OK ) break; + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) break; } } dlrDestroy(&left); dlrDestroy(&right); dlwDestroy(&writer); + return rc; } /* @@ -1394,7 +1490,7 @@ static void docListUnion( ** from pLeft instead of pRight. In the example above, the positions "5" ** and "20" would be added instead of "6" and "21". */ -static void posListPhraseMerge( +static int posListPhraseMerge( DLReader *pLeft, DLReader *pRight, int nNear, @@ -1404,20 +1500,29 @@ static void posListPhraseMerge( PLReader left, right; PLWriter writer; int match = 0; + int rc; assert( dlrDocid(pLeft)==dlrDocid(pRight) ); assert( pOut->iType!=DL_POSITIONS_OFFSETS ); - plrInit(&left, pLeft); - plrInit(&right, pRight); + rc = plrInit(&left, pLeft); + if( rc!=SQLITE_OK ) return rc; + rc = plrInit(&right, pRight); + if( rc!=SQLITE_OK ){ + plrDestroy(&left); + return rc; + } while( !plrAtEnd(&left) && !plrAtEnd(&right) ){ if( plrColumn(&left)<plrColumn(&right) ){ - plrStep(&left); + rc = plrStep(&left); + if( rc!=SQLITE_OK ) break; }else if( plrColumn(&left)>plrColumn(&right) ){ - plrStep(&right); + rc = plrStep(&right); + if( rc!=SQLITE_OK ) break; }else if( plrPosition(&left)>=plrPosition(&right) ){ - plrStep(&right); + rc = plrStep(&right); + if( rc!=SQLITE_OK ) break; }else{ if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){ if( !match ){ @@ -1429,9 +1534,11 @@ static void posListPhraseMerge( }else{ plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0); } - plrStep(&right); + rc = plrStep(&right); + if( rc!=SQLITE_OK ) break; }else{ - plrStep(&left); + rc = plrStep(&left); + if( rc!=SQLITE_OK ) break; } } } @@ -1443,6 +1550,7 @@ static void posListPhraseMerge( plrDestroy(&left); plrDestroy(&right); + return rc; } /* @@ -1496,7 +1604,7 @@ static int plrCompare(PLReader *pLeft, PLReader *pRight){ ** iType controls the type of data written to pOut. If iType is ** DL_POSITIONS, the positions are those from pRight. */ -static void docListPhraseMerge( +static int docListPhraseMerge( const char *pLeft, int nLeft, const char *pRight, int nRight, int nNear, /* 0 for a phrase merge, non-zero for a NEAR merge */ @@ -1506,43 +1614,63 @@ static void docListPhraseMerge( ){ DLReader left, right; DLWriter writer; + int rc; - if( nLeft==0 || nRight==0 ) return; + /* These two buffers are used in the 'while', but are declared here + ** to simplify error-handling. + */ + DataBuffer one = {0, 0, 0}; + DataBuffer two = {0, 0, 0}; + + if( nLeft==0 || nRight==0 ) return SQLITE_OK; assert( iType!=DL_POSITIONS_OFFSETS ); - dlrInit(&left, DL_POSITIONS, pLeft, nLeft); - dlrInit(&right, DL_POSITIONS, pRight, nRight); + rc = dlrInit(&left, DL_POSITIONS, pLeft, nLeft); + if( rc!=SQLITE_OK ) return rc; + rc = dlrInit(&right, DL_POSITIONS, pRight, nRight); + if( rc!=SQLITE_OK ){ + dlrDestroy(&left); + return rc; + } dlwInit(&writer, iType, pOut); while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){ if( dlrDocid(&left)<dlrDocid(&right) ){ - dlrStep(&left); + rc = dlrStep(&left); + if( rc!=SQLITE_OK ) goto err; }else if( dlrDocid(&right)<dlrDocid(&left) ){ - dlrStep(&right); + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) goto err; }else{ if( nNear==0 ){ - posListPhraseMerge(&left, &right, 0, 0, &writer); + rc = posListPhraseMerge(&left, &right, 0, 0, &writer); + if( rc!=SQLITE_OK ) goto err; }else{ /* This case occurs when two terms (simple terms or phrases) are * connected by a NEAR operator, span (nNear+1). i.e. * * '"terrible company" NEAR widget' */ - DataBuffer one = {0, 0, 0}; - DataBuffer two = {0, 0, 0}; - DLWriter dlwriter2; - DLReader dr1 = {0, 0, 0, 0, 0}; + DLReader dr1 = {0, 0, 0, 0, 0}; DLReader dr2 = {0, 0, 0, 0, 0}; dlwInit(&dlwriter2, iType, &one); - posListPhraseMerge(&right, &left, nNear-3+nPhrase, 1, &dlwriter2); + rc = posListPhraseMerge(&right, &left, nNear-3+nPhrase, 1, &dlwriter2); + if( rc!=SQLITE_OK ) goto err; dlwInit(&dlwriter2, iType, &two); - posListPhraseMerge(&left, &right, nNear-1, 0, &dlwriter2); + rc = posListPhraseMerge(&left, &right, nNear-1, 0, &dlwriter2); + if( rc!=SQLITE_OK ) goto err; - if( one.nData) dlrInit(&dr1, iType, one.pData, one.nData); - if( two.nData) dlrInit(&dr2, iType, two.pData, two.nData); + if( one.nData){ + rc = dlrInit(&dr1, iType, one.pData, one.nData); + if( rc!=SQLITE_OK ) goto err; + } + if( two.nData){ + rc = dlrInit(&dr2, iType, two.pData, two.nData); + if( rc!=SQLITE_OK ) goto err; + } if( !dlrAtEnd(&dr1) || !dlrAtEnd(&dr2) ){ PLReader pr1 = {0}; @@ -1551,161 +1679,211 @@ static void docListPhraseMerge( PLWriter plwriter; plwInit(&plwriter, &writer, dlrDocid(dlrAtEnd(&dr1)?&dr2:&dr1)); - if( one.nData ) plrInit(&pr1, &dr1); - if( two.nData ) plrInit(&pr2, &dr2); + if( one.nData ){ + rc = plrInit(&pr1, &dr1); + if( rc!=SQLITE_OK ) goto err; + } + if( two.nData ){ + rc = plrInit(&pr2, &dr2); + if( rc!=SQLITE_OK ) goto err; + } while( !plrAtEnd(&pr1) || !plrAtEnd(&pr2) ){ int iCompare = plrCompare(&pr1, &pr2); switch( iCompare ){ case -1: plwCopy(&plwriter, &pr1); - plrStep(&pr1); + rc = plrStep(&pr1); + if( rc!=SQLITE_OK ) goto err; break; case 1: plwCopy(&plwriter, &pr2); - plrStep(&pr2); + rc = plrStep(&pr2); + if( rc!=SQLITE_OK ) goto err; break; case 0: plwCopy(&plwriter, &pr1); - plrStep(&pr1); - plrStep(&pr2); + rc = plrStep(&pr1); + if( rc!=SQLITE_OK ) goto err; + rc = plrStep(&pr2); + if( rc!=SQLITE_OK ) goto err; break; } } plwTerminate(&plwriter); } - dataBufferDestroy(&one); - dataBufferDestroy(&two); + dataBufferReset(&one); + dataBufferReset(&two); } - dlrStep(&left); - dlrStep(&right); + rc = dlrStep(&left); + if( rc!=SQLITE_OK ) goto err; + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) goto err; } } +err: + dataBufferDestroy(&one); + dataBufferDestroy(&two); dlrDestroy(&left); dlrDestroy(&right); dlwDestroy(&writer); + return rc; } /* We have two DL_DOCIDS doclists: pLeft and pRight. ** Write the intersection of these two doclists into pOut as a ** DL_DOCIDS doclist. */ -static void docListAndMerge( +static int docListAndMerge( const char *pLeft, int nLeft, const char *pRight, int nRight, DataBuffer *pOut /* Write the combined doclist here */ ){ DLReader left, right; DLWriter writer; + int rc; - if( nLeft==0 || nRight==0 ) return; + if( nLeft==0 || nRight==0 ) return SQLITE_OK; - dlrInit(&left, DL_DOCIDS, pLeft, nLeft); - dlrInit(&right, DL_DOCIDS, pRight, nRight); + rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft); + if( rc!=SQLITE_OK ) return rc; + rc = dlrInit(&right, DL_DOCIDS, pRight, nRight); + if( rc!=SQLITE_OK ){ + dlrDestroy(&left); + return rc; + } dlwInit(&writer, DL_DOCIDS, pOut); while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){ if( dlrDocid(&left)<dlrDocid(&right) ){ - dlrStep(&left); + rc = dlrStep(&left); + if( rc!=SQLITE_OK ) break; }else if( dlrDocid(&right)<dlrDocid(&left) ){ - dlrStep(&right); + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) break; }else{ dlwAdd(&writer, dlrDocid(&left)); - dlrStep(&left); - dlrStep(&right); + rc = dlrStep(&left); + if( rc!=SQLITE_OK ) break; + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) break; } } dlrDestroy(&left); dlrDestroy(&right); dlwDestroy(&writer); + return rc; } /* We have two DL_DOCIDS doclists: pLeft and pRight. ** Write the union of these two doclists into pOut as a ** DL_DOCIDS doclist. */ -static void docListOrMerge( +static int docListOrMerge( const char *pLeft, int nLeft, const char *pRight, int nRight, DataBuffer *pOut /* Write the combined doclist here */ ){ DLReader left, right; DLWriter writer; + int rc; if( nLeft==0 ){ if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight); - return; + return SQLITE_OK; } if( nRight==0 ){ dataBufferAppend(pOut, pLeft, nLeft); - return; + return SQLITE_OK; } - dlrInit(&left, DL_DOCIDS, pLeft, nLeft); - dlrInit(&right, DL_DOCIDS, pRight, nRight); + rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft); + if( rc!=SQLITE_OK ) return rc; + rc = dlrInit(&right, DL_DOCIDS, pRight, nRight); + if( rc!=SQLITE_OK ){ + dlrDestroy(&left); + return rc; + } dlwInit(&writer, DL_DOCIDS, pOut); while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){ if( dlrAtEnd(&right) ){ dlwAdd(&writer, dlrDocid(&left)); - dlrStep(&left); + rc = dlrStep(&left); + if( rc!=SQLITE_OK ) break; }else if( dlrAtEnd(&left) ){ dlwAdd(&writer, dlrDocid(&right)); - dlrStep(&right); + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) break; }else if( dlrDocid(&left)<dlrDocid(&right) ){ dlwAdd(&writer, dlrDocid(&left)); - dlrStep(&left); + rc = dlrStep(&left); + if( rc!=SQLITE_OK ) break; }else if( dlrDocid(&right)<dlrDocid(&left) ){ dlwAdd(&writer, dlrDocid(&right)); - dlrStep(&right); + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) break; }else{ dlwAdd(&writer, dlrDocid(&left)); - dlrStep(&left); - dlrStep(&right); + rc = dlrStep(&left); + if( rc!=SQLITE_OK ) break; + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) break; } } dlrDestroy(&left); dlrDestroy(&right); dlwDestroy(&writer); + return rc; } /* We have two DL_DOCIDS doclists: pLeft and pRight. ** Write into pOut as DL_DOCIDS doclist containing all documents that ** occur in pLeft but not in pRight. */ -static void docListExceptMerge( +static int docListExceptMerge( const char *pLeft, int nLeft, const char *pRight, int nRight, DataBuffer *pOut /* Write the combined doclist here */ ){ DLReader left, right; DLWriter writer; + int rc; - if( nLeft==0 ) return; + if( nLeft==0 ) return SQLITE_OK; if( nRight==0 ){ dataBufferAppend(pOut, pLeft, nLeft); - return; + return SQLITE_OK; } - dlrInit(&left, DL_DOCIDS, pLeft, nLeft); - dlrInit(&right, DL_DOCIDS, pRight, nRight); + rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft); + if( rc!=SQLITE_OK ) return rc; + rc = dlrInit(&right, DL_DOCIDS, pRight, nRight); + if( rc!=SQLITE_OK ){ + dlrDestroy(&left); + return rc; + } dlwInit(&writer, DL_DOCIDS, pOut); while( !dlrAtEnd(&left) ){ while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){ - dlrStep(&right); + rc = dlrStep(&right); + if( rc!=SQLITE_OK ) goto err; } if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){ dlwAdd(&writer, dlrDocid(&left)); } - dlrStep(&left); + rc = dlrStep(&left); + if( rc!=SQLITE_OK ) break; } +err: dlrDestroy(&left); dlrDestroy(&right); dlwDestroy(&writer); + return rc; } static char *string_dup_n(const char *s, int n){ @@ -1858,7 +2036,7 @@ static const char *const fulltext_zStatement[MAX_STMT] = { /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?", /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)", /* SEGDIR_SELECT_LEVEL */ - "select start_block, leaves_end_block, root from %_segdir " + "select start_block, leaves_end_block, root, idx from %_segdir " " where level = ? order by idx", /* SEGDIR_SPAN */ "select min(start_block), max(end_block) from %_segdir " @@ -3680,7 +3858,8 @@ static int fulltextNext(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader)); - dlrStep(&c->reader); + if( rc!=SQLITE_OK ) return rc; + rc = dlrStep(&c->reader); if( rc!=SQLITE_OK ) return rc; /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ rc = sqlite3_step(c->pStmt); @@ -3688,8 +3867,10 @@ static int fulltextNext(sqlite3_vtab_cursor *pCursor){ c->eof = 0; return SQLITE_OK; } - /* an error occurred; abort */ - return rc==SQLITE_DONE ? SQLITE_ERROR : rc; + /* Corrupt if the index refers to missing document. */ + if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT; + + return rc; } } @@ -3739,11 +3920,12 @@ static int docListOfPhrase( if( ii==(pPhrase->nToken-1) ){ eType = eListType; } - docListPhraseMerge( + rc = docListPhraseMerge( res.pData, res.nData, tmp.pData, tmp.nData, 0, 0, eType, pResult ); dataBufferDestroy(&res); dataBufferDestroy(&tmp); + if( rc!= SQLITE_OK ) return rc; } } } @@ -3798,21 +3980,21 @@ static int evalFts3Expr( assert( pExpr->pRight->eType==FTSQUERY_PHRASE ); assert( pLeft->eType==FTSQUERY_PHRASE ); nToken = pLeft->pPhrase->nToken + pExpr->pRight->pPhrase->nToken; - docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, + rc = docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pExpr->nNear+1, nToken, eType, pRes ); break; } case FTSQUERY_NOT: { - docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes); + rc = docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes); break; } case FTSQUERY_AND: { - docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); + rc = docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); break; } case FTSQUERY_OR: { - docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); + rc = docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); break; } } @@ -4469,22 +4651,19 @@ static void interiorReaderDestroy(InteriorReader *pReader){ SCRAMBLE(pReader); } -/* TODO(shess) The assertions are great, but what if we're in NDEBUG -** and the blob is empty or otherwise contains suspect data? -*/ -static void interiorReaderInit(const char *pData, int nData, - InteriorReader *pReader){ +static int interiorReaderInit(const char *pData, int nData, + InteriorReader *pReader){ int n, nTerm; - /* Require at least the leading flag byte */ + /* These conditions are checked and met by the callers. */ assert( nData>0 ); assert( pData[0]!='\0' ); CLEAR(pReader); /* Decode the base blockid, and set the cursor to the first term. */ - n = fts3GetVarint(pData+1, &pReader->iBlockid); - assert( 1+n<=nData ); + n = fts3GetVarintSafe(pData+1, &pReader->iBlockid, nData-1); + if( !n ) return SQLITE_CORRUPT_BKPT; pReader->pData = pData+1+n; pReader->nData = nData-(1+n); @@ -4495,17 +4674,18 @@ static void interiorReaderInit(const char *pData, int nData, if( pReader->nData==0 ){ dataBufferInit(&pReader->term, 0); }else{ - n = fts3GetVarint32(pReader->pData, &nTerm); + n = fts3GetVarint32Safe(pReader->pData, &nTerm, pReader->nData); + if( !n || nTerm<0 || nTerm>pReader->nData-n) return SQLITE_CORRUPT_BKPT; dataBufferInit(&pReader->term, nTerm); dataBufferReplace(&pReader->term, pReader->pData+n, nTerm); - assert( n+nTerm<=pReader->nData ); pReader->pData += n+nTerm; pReader->nData -= n+nTerm; } + return SQLITE_OK; } static int interiorReaderAtEnd(InteriorReader *pReader){ - return pReader->term.nData==0; + return pReader->term.nData<=0; } static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){ @@ -4522,7 +4702,7 @@ static const char *interiorReaderTerm(InteriorReader *pReader){ } /* Step forward to the next term in the node. */ -static void interiorReaderStep(InteriorReader *pReader){ +static int interiorReaderStep(InteriorReader *pReader){ assert( !interiorReaderAtEnd(pReader) ); /* If the last term has been read, signal eof, else construct the @@ -4533,18 +4713,26 @@ static void interiorReaderStep(InteriorReader *pReader){ }else{ int n, nPrefix, nSuffix; - n = fts3GetVarint32(pReader->pData, &nPrefix); - n += fts3GetVarint32(pReader->pData+n, &nSuffix); + n = fts3GetVarint32Safe(pReader->pData, &nPrefix, pReader->nData); + if( !n ) return SQLITE_CORRUPT_BKPT; + pReader->nData -= n; + pReader->pData += n; + n += fts3GetVarint32Safe(pReader->pData, &nSuffix, pReader->nData); + if( !n ) return SQLITE_CORRUPT_BKPT; + pReader->nData -= n; + pReader->pData += n; + if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT; + if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT; /* Truncate the current term and append suffix data. */ pReader->term.nData = nPrefix; - dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); + dataBufferAppend(&pReader->term, pReader->pData, nSuffix); - assert( n+nSuffix<=pReader->nData ); - pReader->pData += n+nSuffix; - pReader->nData -= n+nSuffix; + pReader->pData += nSuffix; + pReader->nData -= nSuffix; } pReader->iBlockid++; + return SQLITE_OK; } /* Compare the current term to pTerm[nTerm], returning strcmp-style @@ -4916,7 +5104,8 @@ static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter, n = fts3PutVarint(c, nData); dataBufferAppend(&pWriter->data, c, n); - docListMerge(&pWriter->data, pReaders, nReaders); + rc = docListMerge(&pWriter->data, pReaders, nReaders); + if( rc!=SQLITE_OK ) return rc; ASSERT_VALID_DOCLIST(DL_DEFAULT, pWriter->data.pData+iDoclistData+n, pWriter->data.nData-iDoclistData-n, NULL); @@ -5026,7 +5215,8 @@ static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter, int rc; DLReader reader; - dlrInit(&reader, DL_DEFAULT, pData, nData); + rc = dlrInit(&reader, DL_DEFAULT, pData, nData); + if( rc!=SQLITE_OK ) return rc; rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1); dlrDestroy(&reader); @@ -5071,38 +5261,41 @@ static int leafReaderDataBytes(LeafReader *pReader){ static const char *leafReaderData(LeafReader *pReader){ int n, nData; assert( pReader->term.nData>0 ); - n = fts3GetVarint32(pReader->pData, &nData); + n = fts3GetVarint32Safe(pReader->pData, &nData, pReader->nData); + if( !n || nData>pReader->nData-n ) return NULL; return pReader->pData+n; } -static void leafReaderInit(const char *pData, int nData, - LeafReader *pReader){ +static int leafReaderInit(const char *pData, int nData, + LeafReader *pReader){ int nTerm, n; + /* All callers check this precondition. */ assert( nData>0 ); assert( pData[0]=='\0' ); CLEAR(pReader); /* Read the first term, skipping the header byte. */ - n = fts3GetVarint32(pData+1, &nTerm); + n = fts3GetVarint32Safe(pData+1, &nTerm, nData-1); + if( !n || nTerm<0 || nTerm>nData-1-n ) return SQLITE_CORRUPT_BKPT; dataBufferInit(&pReader->term, nTerm); dataBufferReplace(&pReader->term, pData+1+n, nTerm); /* Position after the first term. */ - assert( 1+n+nTerm<nData ); pReader->pData = pData+1+n+nTerm; pReader->nData = nData-1-n-nTerm; + return SQLITE_OK; } /* Step the reader forward to the next term. */ -static void leafReaderStep(LeafReader *pReader){ +static int leafReaderStep(LeafReader *pReader){ int n, nData, nPrefix, nSuffix; assert( !leafReaderAtEnd(pReader) ); /* Skip previous entry's data block. */ - n = fts3GetVarint32(pReader->pData, &nData); - assert( n+nData<=pReader->nData ); + n = fts3GetVarint32Safe(pReader->pData, &nData, pReader->nData); + if( !n || nData<0 || nData>pReader->nData-n ) return SQLITE_CORRUPT_BKPT; pReader->pData += n+nData; pReader->nData -= n+nData; @@ -5110,15 +5303,23 @@ static void leafReaderStep(LeafReader *pReader){ /* Construct the new term using a prefix from the old term plus a ** suffix from the leaf data. */ - n = fts3GetVarint32(pReader->pData, &nPrefix); - n += fts3GetVarint32(pReader->pData+n, &nSuffix); - assert( n+nSuffix<pReader->nData ); + n = fts3GetVarint32Safe(pReader->pData, &nPrefix, pReader->nData); + if( !n ) return SQLITE_CORRUPT_BKPT; + pReader->nData -= n; + pReader->pData += n; + n = fts3GetVarint32Safe(pReader->pData, &nSuffix, pReader->nData); + if( !n ) return SQLITE_CORRUPT_BKPT; + pReader->nData -= n; + pReader->pData += n; + if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT; + if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT; pReader->term.nData = nPrefix; - dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); + dataBufferAppend(&pReader->term, pReader->pData, nSuffix); - pReader->pData += n+nSuffix; - pReader->nData -= n+nSuffix; + pReader->pData += nSuffix; + pReader->nData -= nSuffix; } + return SQLITE_OK; } /* strcmp-style comparison of pReader's current term against pTerm. @@ -5222,32 +5423,65 @@ static int leavesReaderInit(fulltext_vtab *v, dataBufferInit(&pReader->rootData, 0); if( iStartBlockid==0 ){ + int rc; + /* Corrupt if this can't be a leaf node. */ + if( pRootData==NULL || nRootData<1 || pRootData[0]!='\0' ){ + return SQLITE_CORRUPT_BKPT; + } /* Entire leaf level fit in root data. */ dataBufferReplace(&pReader->rootData, pRootData, nRootData); - leafReaderInit(pReader->rootData.pData, pReader->rootData.nData, - &pReader->leafReader); + rc = leafReaderInit(pReader->rootData.pData, pReader->rootData.nData, + &pReader->leafReader); + if( rc!=SQLITE_OK ){ + dataBufferDestroy(&pReader->rootData); + return rc; + } }else{ sqlite3_stmt *s; int rc = sql_get_leaf_statement(v, idx, &s); if( rc!=SQLITE_OK ) return rc; rc = sqlite3_bind_int64(s, 1, iStartBlockid); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ) goto err; rc = sqlite3_bind_int64(s, 2, iEndBlockid); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ) goto err; rc = sqlite3_step(s); + + /* Corrupt if interior node referenced missing leaf node. */ if( rc==SQLITE_DONE ){ - pReader->eof = 1; - return SQLITE_OK; + rc = SQLITE_CORRUPT_BKPT; + goto err; + } + + if( rc!=SQLITE_ROW ) goto err; + rc = SQLITE_OK; + + /* Corrupt if leaf data isn't a blob. */ + if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + const char *pLeafData = sqlite3_column_blob(s, 0); + int nLeafData = sqlite3_column_bytes(s, 0); + + /* Corrupt if this can't be a leaf node. */ + if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = leafReaderInit(pLeafData, nLeafData, &pReader->leafReader); + } } - if( rc!=SQLITE_ROW ) return rc; - pReader->pStmt = s; - leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0), - sqlite3_column_bytes(pReader->pStmt, 0), - &pReader->leafReader); + err: + if( rc!=SQLITE_OK ){ + if( idx==-1 ){ + sqlite3_finalize(s); + }else{ + sqlite3_reset(s); + } + return rc; + } } return SQLITE_OK; } @@ -5256,11 +5490,12 @@ static int leavesReaderInit(fulltext_vtab *v, ** end of the current leaf, step forward to the next leaf block. */ static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){ + int rc; assert( !leavesReaderAtEnd(pReader) ); - leafReaderStep(&pReader->leafReader); + rc = leafReaderStep(&pReader->leafReader); + if( rc!=SQLITE_OK ) return rc; if( leafReaderAtEnd(&pReader->leafReader) ){ - int rc; if( pReader->rootData.pData ){ pReader->eof = 1; return SQLITE_OK; @@ -5270,10 +5505,25 @@ static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){ pReader->eof = 1; return rc==SQLITE_DONE ? SQLITE_OK : rc; } - leafReaderDestroy(&pReader->leafReader); - leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0), - sqlite3_column_bytes(pReader->pStmt, 0), - &pReader->leafReader); + + /* Corrupt if leaf data isn't a blob. */ + if( sqlite3_column_type(pReader->pStmt, 0)!=SQLITE_BLOB ){ + return SQLITE_CORRUPT_BKPT; + }else{ + LeafReader tmp; + const char *pLeafData = sqlite3_column_blob(pReader->pStmt, 0); + int nLeafData = sqlite3_column_bytes(pReader->pStmt, 0); + + /* Corrupt if this can't be a leaf node. */ + if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){ + return SQLITE_CORRUPT_BKPT; + } + + rc = leafReaderInit(pLeafData, nLeafData, &tmp); + if( rc!=SQLITE_OK ) return rc; + leafReaderDestroy(&pReader->leafReader); + pReader->leafReader = tmp; + } } return SQLITE_OK; } @@ -5334,8 +5584,19 @@ static int leavesReadersInit(fulltext_vtab *v, int iLevel, sqlite_int64 iEnd = sqlite3_column_int64(s, 1); const char *pRootData = sqlite3_column_blob(s, 2); int nRootData = sqlite3_column_bytes(s, 2); + sqlite_int64 iIndex = sqlite3_column_int64(s, 3); + + /* Corrupt if we get back different types than we stored. */ + /* Also corrupt if the index is not sequential starting at 0. */ + if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER || + sqlite3_column_type(s, 1)!=SQLITE_INTEGER || + sqlite3_column_type(s, 2)!=SQLITE_BLOB || + i!=iIndex || + i>=MERGE_COUNT ){ + rc = SQLITE_CORRUPT_BKPT; + break; + } - assert( i<MERGE_COUNT ); rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData, &pReaders[i]); if( rc!=SQLITE_OK ) break; @@ -5346,6 +5607,7 @@ static int leavesReadersInit(fulltext_vtab *v, int iLevel, while( i-->0 ){ leavesReaderDestroy(&pReaders[i]); } + sqlite3_reset(s); /* So we don't leave a lock. */ return rc; } @@ -5369,13 +5631,26 @@ static int leavesReadersMerge(fulltext_vtab *v, DLReader dlReaders[MERGE_COUNT]; const char *pTerm = leavesReaderTerm(pReaders); int i, nTerm = leavesReaderTermBytes(pReaders); + int rc; assert( nReaders<=MERGE_COUNT ); for(i=0; i<nReaders; i++){ - dlrInit(&dlReaders[i], DL_DEFAULT, - leavesReaderData(pReaders+i), - leavesReaderDataBytes(pReaders+i)); + const char *pData = leavesReaderData(pReaders+i); + if( pData==NULL ){ + rc = SQLITE_CORRUPT_BKPT; + break; + } + rc = dlrInit(&dlReaders[i], DL_DEFAULT, + pData, + leavesReaderDataBytes(pReaders+i)); + if( rc!=SQLITE_OK ) break; + } + if( rc!=SQLITE_OK ){ + while( i-->0 ){ + dlrDestroy(&dlReaders[i]); + } + return rc; } return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders); @@ -5429,10 +5704,14 @@ static int segmentMerge(fulltext_vtab *v, int iLevel){ memset(&lrs, '\0', sizeof(lrs)); rc = leavesReadersInit(v, iLevel, lrs, &i); if( rc!=SQLITE_OK ) return rc; - assert( i==MERGE_COUNT ); leafWriterInit(iLevel+1, idx, &writer); + if( i!=MERGE_COUNT ){ + rc = SQLITE_CORRUPT_BKPT; + goto err; + } + /* Since leavesReaderReorder() pushes readers at eof to the end, ** when the first reader is empty, all will be empty. */ @@ -5475,12 +5754,14 @@ static int segmentMerge(fulltext_vtab *v, int iLevel){ } /* Accumulate the union of *acc and *pData into *acc. */ -static void docListAccumulateUnion(DataBuffer *acc, - const char *pData, int nData) { +static int docListAccumulateUnion(DataBuffer *acc, + const char *pData, int nData) { DataBuffer tmp = *acc; + int rc; dataBufferInit(acc, tmp.nData+nData); - docListUnion(tmp.pData, tmp.nData, pData, nData, acc); + rc = docListUnion(tmp.pData, tmp.nData, pData, nData, acc); dataBufferDestroy(&tmp); + return rc; } /* TODO(shess) It might be interesting to explore different merge @@ -5522,8 +5803,13 @@ static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader, int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix); if( c>0 ) break; /* Past any possible matches. */ if( c==0 ){ + int iBuffer, nData; const char *pData = leavesReaderData(pReader); - int iBuffer, nData = leavesReaderDataBytes(pReader); + if( pData==NULL ){ + rc = SQLITE_CORRUPT_BKPT; + break; + } + nData = leavesReaderDataBytes(pReader); /* Find the first empty buffer. */ for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){ @@ -5569,11 +5855,13 @@ static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader, ** with pData/nData. */ dataBufferSwap(p, pAcc); - docListAccumulateUnion(pAcc, pData, nData); + rc = docListAccumulateUnion(pAcc, pData, nData); + if( rc!=SQLITE_OK ) goto err; /* Accumulate remaining doclists into pAcc. */ for(++p; p<pAcc; ++p){ - docListAccumulateUnion(pAcc, p->pData, p->nData); + rc = docListAccumulateUnion(pAcc, p->pData, p->nData); + if( rc!=SQLITE_OK ) goto err; /* dataBufferReset() could allow a large doclist to blow up ** our memory requirements. @@ -5598,13 +5886,15 @@ static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader, if( out->nData==0 ){ dataBufferSwap(out, &(pBuffers[iBuffer])); }else{ - docListAccumulateUnion(out, pBuffers[iBuffer].pData, - pBuffers[iBuffer].nData); + rc = docListAccumulateUnion(out, pBuffers[iBuffer].pData, + pBuffers[iBuffer].nData); + if( rc!=SQLITE_OK ) break; } } } } +err: while( nBuffers-- ){ dataBufferDestroy(&(pBuffers[nBuffers])); } @@ -5663,20 +5953,26 @@ static int loadSegmentLeaves(fulltext_vtab *v, ** node. Consider whether breaking symmetry is worthwhile. I suspect ** it is not worthwhile. */ -static void getChildrenContaining(const char *pData, int nData, - const char *pTerm, int nTerm, int isPrefix, - sqlite_int64 *piStartChild, - sqlite_int64 *piEndChild){ +static int getChildrenContaining(const char *pData, int nData, + const char *pTerm, int nTerm, int isPrefix, + sqlite_int64 *piStartChild, + sqlite_int64 *piEndChild){ InteriorReader reader; + int rc; assert( nData>1 ); assert( *pData!='\0' ); - interiorReaderInit(pData, nData, &reader); + rc = interiorReaderInit(pData, nData, &reader); + if( rc!=SQLITE_OK ) return rc; /* Scan for the first child which could contain pTerm/nTerm. */ while( !interiorReaderAtEnd(&reader) ){ if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break; - interiorReaderStep(&reader); + rc = interiorReaderStep(&reader); + if( rc!=SQLITE_OK ){ + interiorReaderDestroy(&reader); + return rc; + } } *piStartChild = interiorReaderCurrentBlockid(&reader); @@ -5686,7 +5982,11 @@ static void getChildrenContaining(const char *pData, int nData, */ while( !interiorReaderAtEnd(&reader) ){ if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break; - interiorReaderStep(&reader); + rc = interiorReaderStep(&reader); + if( rc!=SQLITE_OK ){ + interiorReaderDestroy(&reader); + return rc; + } } *piEndChild = interiorReaderCurrentBlockid(&reader); @@ -5695,6 +5995,7 @@ static void getChildrenContaining(const char *pData, int nData, /* Children must ascend, and if !prefix, both must be the same. */ assert( *piEndChild>=*piStartChild ); assert( isPrefix || *piStartChild==*piEndChild ); + return rc; } /* Read block at iBlockid and pass it with other params to @@ -5722,11 +6023,31 @@ static int loadAndGetChildrenContaining( if( rc!=SQLITE_OK ) return rc; rc = sqlite3_step(s); - if( rc==SQLITE_DONE ) return SQLITE_ERROR; + /* Corrupt if interior node references missing child node. */ + if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT; if( rc!=SQLITE_ROW ) return rc; - getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0), - pTerm, nTerm, isPrefix, piStartChild, piEndChild); + /* Corrupt if child node isn't a blob. */ + if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){ + sqlite3_reset(s); /* So we don't leave a lock. */ + return SQLITE_CORRUPT_BKPT; + }else{ + const char *pData = sqlite3_column_blob(s, 0); + int nData = sqlite3_column_bytes(s, 0); + + /* Corrupt if child is not a valid interior node. */ + if( pData==NULL || nData<1 || pData[0]=='\0' ){ + sqlite3_reset(s); /* So we don't leave a lock. */ + return SQLITE_CORRUPT_BKPT; + } + + rc = getChildrenContaining(pData, nData, pTerm, nTerm, + isPrefix, piStartChild, piEndChild); + if( rc!=SQLITE_OK ){ + sqlite3_reset(s); + return rc; + } + } /* We expect only one row. We must execute another sqlite3_step() * to complete the iteration; otherwise the table will remain @@ -5756,8 +6077,9 @@ static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData, /* Process pData as an interior node, then loop down the tree ** until we find the set of leaf nodes to scan for the term. */ - getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix, - &iStartChild, &iEndChild); + rc = getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix, + &iStartChild, &iEndChild); + if( rc!=SQLITE_OK ) return rc; while( iStartChild>iLeavesEnd ){ sqlite_int64 iNextStart, iNextEnd; rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix, @@ -5809,7 +6131,8 @@ static int loadSegment(fulltext_vtab *v, const char *pData, int nData, DataBuffer result; int rc; - assert( nData>1 ); + /* Corrupt if segment root can't be valid. */ + if( pData==NULL || nData<1 ) return SQLITE_CORRUPT_BKPT; /* This code should never be called with buffered updates. */ assert( v->nPendingData<0 ); @@ -5826,16 +6149,21 @@ static int loadSegment(fulltext_vtab *v, const char *pData, int nData, DataBuffer merged; DLReader readers[2]; - dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData); - dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData); - dataBufferInit(&merged, out->nData+result.nData); - docListMerge(&merged, readers, 2); - dataBufferDestroy(out); - *out = merged; - dlrDestroy(&readers[0]); - dlrDestroy(&readers[1]); + rc = dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData); + if( rc==SQLITE_OK ){ + rc = dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData); + if( rc==SQLITE_OK ){ + dataBufferInit(&merged, out->nData+result.nData); + rc = docListMerge(&merged, readers, 2); + dataBufferDestroy(out); + *out = merged; + dlrDestroy(&readers[1]); + } + dlrDestroy(&readers[0]); + } } } + dataBufferDestroy(&result); return rc; } @@ -5869,11 +6197,20 @@ static int termSelect( const char *pData = sqlite3_column_blob(s, 2); const int nData = sqlite3_column_bytes(s, 2); const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); + + /* Corrupt if we get back different types than we stored. */ + if( sqlite3_column_type(s, 1)!=SQLITE_INTEGER || + sqlite3_column_type(s, 2)!=SQLITE_BLOB ){ + rc = SQLITE_CORRUPT_BKPT; + goto err; + } + rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix, &doclist); if( rc!=SQLITE_OK ) goto err; } if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; if( doclist.nData!=0 ){ /* TODO(shess) The old term_select_all() code applied the column ** restrict as we merged segments, leading to smaller buffers. @@ -5881,13 +6218,13 @@ static int termSelect( ** system is checked in. */ if( iColumn==v->nColumn) iColumn = -1; - docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, - iColumn, iType, out); + rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, + iColumn, iType, out); } - rc = SQLITE_OK; } err: + sqlite3_reset(s); /* So we don't leave a lock. */ dataBufferDestroy(&doclist); return rc; } @@ -6250,6 +6587,7 @@ static int optimizeInternal(fulltext_vtab *v, LeafWriter *pWriter){ int i, rc = SQLITE_OK; DataBuffer doclist, merged, tmp; + const char *pData; /* Order the readers. */ i = nReaders; @@ -6270,14 +6608,20 @@ static int optimizeInternal(fulltext_vtab *v, if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break; } + pData = optLeavesReaderData(&readers[0]); + if( pData==NULL ){ + rc = SQLITE_CORRUPT_BKPT; + break; + } + /* Special-case for no merge. */ if( i==1 ){ /* Trim deletions from the doclist. */ dataBufferReset(&merged); - docListTrim(DL_DEFAULT, - optLeavesReaderData(&readers[0]), - optLeavesReaderDataBytes(&readers[0]), - -1, DL_DEFAULT, &merged); + rc = docListTrim(DL_DEFAULT, pData, + optLeavesReaderDataBytes(&readers[0]), + -1, DL_DEFAULT, &merged); + if( rc!=SQLITE_OK ) break; }else{ DLReader dlReaders[MERGE_COUNT]; int iReader, nReaders; @@ -6285,9 +6629,10 @@ static int optimizeInternal(fulltext_vtab *v, /* Prime the pipeline with the first reader's doclist. After ** one pass index 0 will reference the accumulated doclist. */ - dlrInit(&dlReaders[0], DL_DEFAULT, - optLeavesReaderData(&readers[0]), - optLeavesReaderDataBytes(&readers[0])); + rc = dlrInit(&dlReaders[0], DL_DEFAULT, + pData, + optLeavesReaderDataBytes(&readers[0])); + if( rc!=SQLITE_OK ) break; iReader = 1; assert( iReader<i ); /* Must execute the loop at least once. */ @@ -6295,24 +6640,34 @@ static int optimizeInternal(fulltext_vtab *v, /* Merge 16 inputs per pass. */ for( nReaders=1; iReader<i && nReaders<MERGE_COUNT; iReader++, nReaders++ ){ - dlrInit(&dlReaders[nReaders], DL_DEFAULT, - optLeavesReaderData(&readers[iReader]), - optLeavesReaderDataBytes(&readers[iReader])); + pData = optLeavesReaderData(&readers[iReader]); + if( pData==NULL ){ + rc = SQLITE_CORRUPT_BKPT; + break; + } + rc = dlrInit(&dlReaders[nReaders], DL_DEFAULT, pData, + optLeavesReaderDataBytes(&readers[iReader])); + if( rc!=SQLITE_OK ) break; } /* Merge doclists and swap result into accumulator. */ - dataBufferReset(&merged); - docListMerge(&merged, dlReaders, nReaders); - tmp = merged; - merged = doclist; - doclist = tmp; + if( rc==SQLITE_OK ){ + dataBufferReset(&merged); + rc = docListMerge(&merged, dlReaders, nReaders); + tmp = merged; + merged = doclist; + doclist = tmp; + } while( nReaders-- > 0 ){ dlrDestroy(&dlReaders[nReaders]); } + if( rc!=SQLITE_OK ) goto err; + /* Accumulated doclist to reader 0 for next pass. */ - dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData); + rc = dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData); + if( rc!=SQLITE_OK ) goto err; } /* Destroy reader that was left in the pipeline. */ @@ -6320,8 +6675,9 @@ static int optimizeInternal(fulltext_vtab *v, /* Trim deletions from the doclist. */ dataBufferReset(&merged); - docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, - -1, DL_DEFAULT, &merged); + rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, + -1, DL_DEFAULT, &merged); + if( rc!=SQLITE_OK ) goto err; } /* Only pass doclists with hits (skip if all hits deleted). */ @@ -6401,6 +6757,14 @@ static void optimizeFunc(sqlite3_context *pContext, const char *pRootData = sqlite3_column_blob(s, 2); int nRootData = sqlite3_column_bytes(s, 2); + /* Corrupt if we get back different types than we stored. */ + if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER || + sqlite3_column_type(s, 1)!=SQLITE_INTEGER || + sqlite3_column_type(s, 2)!=SQLITE_BLOB ){ + rc = SQLITE_CORRUPT_BKPT; + break; + } + assert( i<nReaders ); rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData, &readers[i].reader); @@ -6414,6 +6778,8 @@ static void optimizeFunc(sqlite3_context *pContext, if( rc==SQLITE_DONE ){ assert( i==nReaders ); rc = optimizeInternal(v, readers, nReaders, &writer); + }else{ + sqlite3_reset(s); /* So we don't leave a lock. */ } while( i-- > 0 ){ @@ -6477,9 +6843,18 @@ static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s, const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1); const char *pRootData = sqlite3_column_blob(s, 2); const int nRootData = sqlite3_column_bytes(s, 2); + int rc; LeavesReader reader; - int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, - pRootData, nRootData, &reader); + + /* Corrupt if we get back different types than we stored. */ + if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER || + sqlite3_column_type(s, 1)!=SQLITE_INTEGER || + sqlite3_column_type(s, 2)!=SQLITE_BLOB ){ + return SQLITE_CORRUPT_BKPT; + } + + rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, + pRootData, nRootData, &reader); if( rc!=SQLITE_OK ) return rc; while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){ @@ -6641,16 +7016,19 @@ static void createDoclistResult(sqlite3_context *pContext, const char *pData, int nData){ DataBuffer dump; DLReader dlReader; + int rc; assert( pData!=NULL && nData>0 ); + rc = dlrInit(&dlReader, DL_DEFAULT, pData, nData); + if( rc!=SQLITE_OK ) return rc; dataBufferInit(&dump, 0); - dlrInit(&dlReader, DL_DEFAULT, pData, nData); - for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){ + for( ; rc==SQLITE_OK && !dlrAtEnd(&dlReader); rc = dlrStep(&dlReader) ){ char buf[256]; PLReader plReader; - plrInit(&plReader, &dlReader); + rc = plrInit(&plReader, &dlReader); + if( rc!=SQLITE_OK ) break; if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){ sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader)); dataBufferAppend(&dump, buf, strlen(buf)); @@ -6661,7 +7039,8 @@ static void createDoclistResult(sqlite3_context *pContext, dlrDocid(&dlReader), iColumn); dataBufferAppend(&dump, buf, strlen(buf)); - for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){ + for( ; !plrAtEnd(&plReader); rc = plrStep(&plReader) ){ + if( rc!=SQLITE_OK ) break; if( plrColumn(&plReader)!=iColumn ){ iColumn = plrColumn(&plReader); sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn); @@ -6682,6 +7061,7 @@ static void createDoclistResult(sqlite3_context *pContext, dataBufferAppend(&dump, buf, strlen(buf)); } plrDestroy(&plReader); + if( rc!= SQLITE_OK ) break; assert( dump.nData>0 ); dump.nData--; /* Overwrite trailing space. */ @@ -6690,6 +7070,10 @@ static void createDoclistResult(sqlite3_context *pContext, } } dlrDestroy(&dlReader); + if( rc!=SQLITE_OK ){ + dataBufferDestroy(&dump); + return rc; + } assert( dump.nData>0 ); dump.nData--; /* Overwrite trailing space. */ @@ -6701,6 +7085,7 @@ static void createDoclistResult(sqlite3_context *pContext, sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free); dump.pData = NULL; dump.nData = dump.nCapacity = 0; + return SQLITE_OK; } /* Implements dump_doclist() for use in inspecting the fts3 index from @@ -6987,7 +7372,11 @@ int sqlite3Fts3Init(sqlite3 *db){ ** module with sqlite. */ if( SQLITE_OK==rc +#if CHROMIUM_FTS3_CHANGES && !SQLITE_TEST + /* fts3_tokenizer() disabled for security reasons. */ +#else && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) +#endif && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1)) |