diff options
Diffstat (limited to 'third_party/sqlite/fts3.patch')
| -rw-r--r-- | third_party/sqlite/fts3.patch | 1922 |
1 files changed, 1901 insertions, 21 deletions
diff --git a/third_party/sqlite/fts3.patch b/third_party/sqlite/fts3.patch index 0910ea3..622626a 100644 --- a/third_party/sqlite/fts3.patch +++ b/third_party/sqlite/fts3.patch @@ -1,9 +1,17 @@ -diff --git a/third_party/sqlite/src/ext/fts3/fts3.c b/third_party/sqlite/src/ext/fts3/fts3.c -index 5323a5c..d591270 100644 ---- a/third_party/sqlite/src/ext/fts3/fts3.c -+++ b/third_party/sqlite/src/ext/fts3/fts3.c -@@ -313,6 +313,16 @@ - SQLITE_EXTENSION_INIT1 +Index: ext/fts3/fts3.c +=================================================================== +--- ext/fts3/fts3.c (revision 48811) ++++ ext/fts3/fts3.c (working copy) +@@ -271,6 +271,7 @@ + ** deletions and duplications. This would basically be a forced merge + ** into a single segment. + */ ++#define CHROMIUM_FTS3_CHANGES 1 + + #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +@@ -313,6 +314,16 @@ + # define FTSTRACE(A) #endif +#if 0 @@ -16,14 +24,1886 @@ index 5323a5c..d591270 100644 +# define SQLITE_CORRUPT_BKPT SQLITE_CORRUPT +#endif + + /* It is not safe to call isspace(), tolower(), or isalnum() on + ** hi-bit-set characters. This is the same solution used in the + ** tokenizer. +@@ -401,30 +412,41 @@ + /* Read a 64-bit variable-length integer from memory starting at p[0]. + * Return the number of bytes read, or 0 on error. + * The value is stored in *v. */ +-static int fts3GetVarint(const char *p, sqlite_int64 *v){ ++static int fts3GetVarintSafe(const char *p, sqlite_int64 *v, int max){ + const unsigned char *q = (const unsigned char *) p; + sqlite_uint64 x = 0, y = 1; +- while( (*q & 0x80) == 0x80 ){ ++ if( max>VARINT_MAX ) max = VARINT_MAX; ++ while( max && (*q & 0x80) == 0x80 ){ ++ max--; + x += y * (*q++ & 0x7f); + y <<= 7; +- if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */ +- assert( 0 ); +- return 0; +- } + } ++ if( !max ){ ++ assert( 0 ); ++ return 0; /* tried to read too much; bad data */ ++ } + x += y * (*q++); + *v = (sqlite_int64) x; + return (int) (q - (unsigned char *)p); + } + +-static int fts3GetVarint32(const char *p, int *pi){ ++static int fts3GetVarint(const char *p, sqlite_int64 *v){ ++ return fts3GetVarintSafe(p, v, VARINT_MAX); ++} ++ ++static int fts3GetVarint32Safe(const char *p, int *pi, int max){ + sqlite_int64 i; +- int ret = fts3GetVarint(p, &i); ++ int ret = fts3GetVarintSafe(p, &i, max); ++ if( !ret ) return ret; + *pi = (int) i; + assert( *pi==i ); + return ret; + } + ++static int fts3GetVarint32(const char* p, int *pi){ ++ return fts3GetVarint32Safe(p, pi, VARINT_MAX); ++} ++ + /*******************************************************************/ + /* DataBuffer is used to collect data into a buffer in piecemeal + ** fashion. It implements the usual distinction between amount of +@@ -593,7 +615,7 @@ + + static int dlrAtEnd(DLReader *pReader){ + assert( pReader->nData>=0 ); +- return pReader->nData==0; ++ return pReader->nData<=0; + } + static sqlite_int64 dlrDocid(DLReader *pReader){ + assert( !dlrAtEnd(pReader) ); +@@ -617,7 +639,8 @@ + */ + static const char *dlrPosData(DLReader *pReader){ + sqlite_int64 iDummy; +- int n = fts3GetVarint(pReader->pData, &iDummy); ++ int n = fts3GetVarintSafe(pReader->pData, &iDummy, pReader->nElement); ++ if( !n ) return NULL; + assert( !dlrAtEnd(pReader) ); + return pReader->pData+n; + } +@@ -627,7 +650,7 @@ + assert( !dlrAtEnd(pReader) ); + return pReader->nElement-n; + } +-static void dlrStep(DLReader *pReader){ ++static int dlrStep(DLReader *pReader){ + assert( !dlrAtEnd(pReader) ); + + /* Skip past current doclist element. */ +@@ -636,32 +659,44 @@ + pReader->nData -= pReader->nElement; + + /* If there is more data, read the next doclist element. */ +- if( pReader->nData!=0 ){ ++ if( pReader->nData>0 ){ + sqlite_int64 iDocidDelta; +- int iDummy, n = fts3GetVarint(pReader->pData, &iDocidDelta); ++ int nTotal = 0; ++ int iDummy, n = fts3GetVarintSafe(pReader->pData, &iDocidDelta, pReader->nData); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ nTotal += n; + pReader->iDocid += iDocidDelta; + if( pReader->iType>=DL_POSITIONS ){ +- assert( n<pReader->nData ); + while( 1 ){ +- n += fts3GetVarint32(pReader->pData+n, &iDummy); +- assert( n<=pReader->nData ); ++ n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ nTotal += n; + if( iDummy==POS_END ) break; + if( iDummy==POS_COLUMN ){ +- n += fts3GetVarint32(pReader->pData+n, &iDummy); +- assert( n<pReader->nData ); ++ n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ nTotal += n; + }else if( pReader->iType==DL_POSITIONS_OFFSETS ){ +- n += fts3GetVarint32(pReader->pData+n, &iDummy); +- n += fts3GetVarint32(pReader->pData+n, &iDummy); +- assert( n<pReader->nData ); ++ n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ nTotal += n; ++ n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ nTotal += n; + } + } + } +- pReader->nElement = n; ++ pReader->nElement = nTotal; + assert( pReader->nElement<=pReader->nData ); + } ++ return SQLITE_OK; + } +-static void dlrInit(DLReader *pReader, DocListType iType, +- const char *pData, int nData){ ++static void dlrDestroy(DLReader *pReader){ ++ SCRAMBLE(pReader); ++} ++static int dlrInit(DLReader *pReader, DocListType iType, ++ const char *pData, int nData){ ++ int rc; + assert( pData!=NULL && nData!=0 ); + pReader->iType = iType; + pReader->pData = pData; +@@ -670,11 +705,10 @@ + pReader->iDocid = 0; + + /* Load the first element's data. There must be a first element. */ +- dlrStep(pReader); ++ rc = dlrStep(pReader); ++ if( rc!=SQLITE_OK ) dlrDestroy(pReader); ++ return rc; + } +-static void dlrDestroy(DLReader *pReader){ +- SCRAMBLE(pReader); +-} + + #ifndef NDEBUG + /* Verify that the doclist can be validly decoded. Also returns the +@@ -760,9 +794,9 @@ + /* TODO(shess) This has become just a helper for docListMerge. + ** Consider a refactor to make this cleaner. + */ +-static void dlwAppend(DLWriter *pWriter, +- const char *pData, int nData, +- sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){ ++static int dlwAppend(DLWriter *pWriter, ++ const char *pData, int nData, ++ sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){ + sqlite_int64 iDocid = 0; + char c[VARINT_MAX]; + int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */ +@@ -771,7 +805,8 @@ + #endif + + /* Recode the initial docid as delta from iPrevDocid. */ +- nFirstOld = fts3GetVarint(pData, &iDocid); ++ nFirstOld = fts3GetVarintSafe(pData, &iDocid, nData); ++ if( !nFirstOld ) return SQLITE_CORRUPT_BKPT; + assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) ); + nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid); + +@@ -792,10 +827,11 @@ + dataBufferAppend(pWriter->b, c, nFirstNew); + } + pWriter->iPrevDocid = iLastDocid; ++ return SQLITE_OK; + } +-static void dlwCopy(DLWriter *pWriter, DLReader *pReader){ +- dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader), +- dlrDocid(pReader), dlrDocid(pReader)); ++static int dlwCopy(DLWriter *pWriter, DLReader *pReader){ ++ return dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader), ++ dlrDocid(pReader), dlrDocid(pReader)); + } + static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){ + char c[VARINT_MAX]; +@@ -856,45 +892,61 @@ + assert( !plrAtEnd(pReader) ); + return pReader->iEndOffset; + } +-static void plrStep(PLReader *pReader){ +- int i, n; ++static int plrStep(PLReader *pReader){ ++ int i, n, nTotal = 0; + + assert( !plrAtEnd(pReader) ); + +- if( pReader->nData==0 ){ ++ if( pReader->nData<=0 ){ + pReader->pData = NULL; +- return; ++ return SQLITE_OK; + } + +- n = fts3GetVarint32(pReader->pData, &i); ++ n = fts3GetVarint32Safe(pReader->pData, &i, pReader->nData); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ nTotal += n; + if( i==POS_COLUMN ){ +- n += fts3GetVarint32(pReader->pData+n, &pReader->iColumn); ++ n = fts3GetVarint32Safe(pReader->pData+nTotal, &pReader->iColumn, pReader->nData-nTotal); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ nTotal += n; + pReader->iPosition = 0; + pReader->iStartOffset = 0; +- n += fts3GetVarint32(pReader->pData+n, &i); ++ n = fts3GetVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ nTotal += n; + } + /* Should never see adjacent column changes. */ + assert( i!=POS_COLUMN ); + + if( i==POS_END ){ ++ assert( nTotal<=pReader->nData ); + pReader->nData = 0; + pReader->pData = NULL; +- return; ++ return SQLITE_OK; + } + + pReader->iPosition += i-POS_BASE; + if( pReader->iType==DL_POSITIONS_OFFSETS ){ +- n += fts3GetVarint32(pReader->pData+n, &i); ++ n = fts3GetVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ nTotal += n; + pReader->iStartOffset += i; +- n += fts3GetVarint32(pReader->pData+n, &i); ++ n = fts3GetVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ nTotal += n; + pReader->iEndOffset = pReader->iStartOffset+i; + } +- assert( n<=pReader->nData ); +- pReader->pData += n; +- pReader->nData -= n; ++ assert( nTotal<=pReader->nData ); ++ pReader->pData += nTotal; ++ pReader->nData -= nTotal; ++ return SQLITE_OK; + } + +-static void plrInit(PLReader *pReader, DLReader *pDLReader){ ++static void plrDestroy(PLReader *pReader){ ++ SCRAMBLE(pReader); ++} ++static int plrInit(PLReader *pReader, DLReader *pDLReader){ ++ int rc; + pReader->pData = dlrPosData(pDLReader); + pReader->nData = dlrPosDataLen(pDLReader); + pReader->iType = pDLReader->iType; +@@ -902,11 +954,10 @@ + pReader->iPosition = 0; + pReader->iStartOffset = 0; + pReader->iEndOffset = 0; +- plrStep(pReader); ++ rc = plrStep(pReader); ++ if( rc!=SQLITE_OK ) plrDestroy(pReader); ++ return rc; + } +-static void plrDestroy(PLReader *pReader){ +- SCRAMBLE(pReader); +-} + + /*******************************************************************/ + /* PLWriter is used in constructing a document's position list. As a +@@ -1091,14 +1142,16 @@ + ** deletion will be trimmed, and will thus not effect a deletion + ** during the merge. + */ +-static void docListTrim(DocListType iType, const char *pData, int nData, +- int iColumn, DocListType iOutType, DataBuffer *out){ ++static int docListTrim(DocListType iType, const char *pData, int nData, ++ int iColumn, DocListType iOutType, DataBuffer *out){ + DLReader dlReader; + DLWriter dlWriter; ++ int rc; + + assert( iOutType<=iType ); + +- dlrInit(&dlReader, iType, pData, nData); ++ rc = dlrInit(&dlReader, iType, pData, nData); ++ if( rc!=SQLITE_OK ) return rc; + dlwInit(&dlWriter, iOutType, out); + + while( !dlrAtEnd(&dlReader) ){ +@@ -1106,7 +1159,8 @@ + PLWriter plWriter; + int match = 0; + +- plrInit(&plReader, &dlReader); ++ rc = plrInit(&plReader, &dlReader); ++ if( rc!=SQLITE_OK ) break; + + while( !plrAtEnd(&plReader) ){ + if( iColumn==-1 || plrColumn(&plReader)==iColumn ){ +@@ -1117,7 +1171,11 @@ + plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader), + plrStartOffset(&plReader), plrEndOffset(&plReader)); + } +- plrStep(&plReader); ++ rc = plrStep(&plReader); ++ if( rc!=SQLITE_OK ){ ++ plrDestroy(&plReader); ++ goto err; ++ } + } + if( match ){ + plwTerminate(&plWriter); +@@ -1125,10 +1183,13 @@ + } + + plrDestroy(&plReader); +- dlrStep(&dlReader); ++ rc = dlrStep(&dlReader); ++ if( rc!=SQLITE_OK ) break; + } ++err: + dlwDestroy(&dlWriter); + dlrDestroy(&dlReader); ++ return rc; + } + + /* Used by docListMerge() to keep doclists in the ascending order by +@@ -1185,19 +1246,20 @@ + /* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably + ** be fixed. + */ +-static void docListMerge(DataBuffer *out, +- DLReader *pReaders, int nReaders){ ++static int docListMerge(DataBuffer *out, ++ DLReader *pReaders, int nReaders){ + OrderedDLReader readers[MERGE_COUNT]; + DLWriter writer; + int i, n; + const char *pStart = 0; + int nStart = 0; + sqlite_int64 iFirstDocid = 0, iLastDocid = 0; ++ int rc = SQLITE_OK; + + assert( nReaders>0 ); + if( nReaders==1 ){ + dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders)); +- return; ++ return SQLITE_OK; + } + + assert( nReaders<=MERGE_COUNT ); +@@ -1230,20 +1292,23 @@ + nStart += dlrDocDataBytes(readers[0].pReader); + }else{ + if( pStart!=0 ){ +- dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); ++ rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); ++ if( rc!=SQLITE_OK ) goto err; + } + pStart = dlrDocData(readers[0].pReader); + nStart = dlrDocDataBytes(readers[0].pReader); + iFirstDocid = iDocid; + } + iLastDocid = iDocid; +- dlrStep(readers[0].pReader); ++ rc = dlrStep(readers[0].pReader); ++ if( rc!= SQLITE_OK ) goto err; + + /* Drop all of the older elements with the same docid. */ + for(i=1; i<nReaders && + !dlrAtEnd(readers[i].pReader) && + dlrDocid(readers[i].pReader)==iDocid; i++){ +- dlrStep(readers[i].pReader); ++ rc = dlrStep(readers[i].pReader); ++ if( rc!=SQLITE_OK ) goto err; + } + + /* Get the readers back into order. */ +@@ -1253,8 +1318,10 @@ + } + + /* Copy over any remaining elements. */ +- if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); ++ if( nStart>0 ) rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); ++err: + dlwDestroy(&writer); ++ return rc; + } + + /* Helper function for posListUnion(). Compares the current position +@@ -1290,30 +1357,40 @@ + ** work with any doclist type, though both inputs and the output + ** should be the same type. + */ +-static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){ ++static int posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){ + PLReader left, right; + PLWriter writer; ++ int rc; + + assert( dlrDocid(pLeft)==dlrDocid(pRight) ); + assert( pLeft->iType==pRight->iType ); + assert( pLeft->iType==pOut->iType ); + +- plrInit(&left, pLeft); +- plrInit(&right, pRight); ++ rc = plrInit(&left, pLeft); ++ if( rc!=SQLITE_OK ) return rc; ++ rc = plrInit(&right, pRight); ++ if( rc!=SQLITE_OK ){ ++ plrDestroy(&left); ++ return rc; ++ } + plwInit(&writer, pOut, dlrDocid(pLeft)); + + while( !plrAtEnd(&left) || !plrAtEnd(&right) ){ + int c = posListCmp(&left, &right); + if( c<0 ){ + plwCopy(&writer, &left); +- plrStep(&left); ++ rc = plrStep(&left); ++ if( rc!=SQLITE_OK ) break; + }else if( c>0 ){ + plwCopy(&writer, &right); +- plrStep(&right); ++ rc = plrStep(&right); ++ if( rc!=SQLITE_OK ) break; + }else{ + plwCopy(&writer, &left); +- plrStep(&left); +- plrStep(&right); ++ rc = plrStep(&left); ++ if( rc!=SQLITE_OK ) break; ++ rc = plrStep(&right); ++ if( rc!=SQLITE_OK ) break; + } + } + +@@ -1321,56 +1398,75 @@ + plwDestroy(&writer); + plrDestroy(&left); + plrDestroy(&right); ++ return rc; + } + + /* Write the union of doclists in pLeft and pRight to pOut. For + ** docids in common between the inputs, the union of the position + ** lists is written. Inputs and outputs are always type DL_DEFAULT. + */ +-static void docListUnion( ++static int docListUnion( + const char *pLeft, int nLeft, + const char *pRight, int nRight, + DataBuffer *pOut /* Write the combined doclist here */ + ){ + DLReader left, right; + DLWriter writer; ++ int rc; + + if( nLeft==0 ){ + if( nRight!=0) dataBufferAppend(pOut, pRight, nRight); +- return; ++ return SQLITE_OK; + } + if( nRight==0 ){ + dataBufferAppend(pOut, pLeft, nLeft); +- return; ++ return SQLITE_OK; + } + +- dlrInit(&left, DL_DEFAULT, pLeft, nLeft); +- dlrInit(&right, DL_DEFAULT, pRight, nRight); ++ rc = dlrInit(&left, DL_DEFAULT, pLeft, nLeft); ++ if( rc!=SQLITE_OK ) return rc; ++ rc = dlrInit(&right, DL_DEFAULT, pRight, nRight); ++ if( rc!=SQLITE_OK){ ++ dlrDestroy(&left); ++ return rc; ++ } + dlwInit(&writer, DL_DEFAULT, pOut); + + while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){ + if( dlrAtEnd(&right) ){ +- dlwCopy(&writer, &left); +- dlrStep(&left); ++ rc = dlwCopy(&writer, &left); ++ if( rc!=SQLITE_OK) break; ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK) break; + }else if( dlrAtEnd(&left) ){ +- dlwCopy(&writer, &right); +- dlrStep(&right); ++ rc = dlwCopy(&writer, &right); ++ if( rc!=SQLITE_OK ) break; ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) break; + }else if( dlrDocid(&left)<dlrDocid(&right) ){ +- dlwCopy(&writer, &left); +- dlrStep(&left); ++ rc = dlwCopy(&writer, &left); ++ if( rc!=SQLITE_OK ) break; ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK ) break; + }else if( dlrDocid(&left)>dlrDocid(&right) ){ +- dlwCopy(&writer, &right); +- dlrStep(&right); ++ rc = dlwCopy(&writer, &right); ++ if( rc!=SQLITE_OK ) break; ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) break; + }else{ +- posListUnion(&left, &right, &writer); +- dlrStep(&left); +- dlrStep(&right); ++ rc = posListUnion(&left, &right, &writer); ++ if( rc!=SQLITE_OK ) break; ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK ) break; ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) break; + } + } + + dlrDestroy(&left); + dlrDestroy(&right); + dlwDestroy(&writer); ++ return rc; + } + /* - ** Write a 64-bit variable-length integer to memory starting at p[0]. - ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. -diff --git a/third_party/sqlite/src/ext/fts3/fts3_icu.c b/third_party/sqlite/src/ext/fts3/fts3_icu.c -index 85390d3..a75b14a 100644 ---- a/third_party/sqlite/src/ext/fts3/fts3_icu.c -+++ b/third_party/sqlite/src/ext/fts3/fts3_icu.c -@@ -198,7 +198,7 @@ static int icuNext( +@@ -1394,7 +1490,7 @@ + ** from pLeft instead of pRight. In the example above, the positions "5" + ** and "20" would be added instead of "6" and "21". + */ +-static void posListPhraseMerge( ++static int posListPhraseMerge( + DLReader *pLeft, + DLReader *pRight, + int nNear, +@@ -1404,20 +1500,29 @@ + PLReader left, right; + PLWriter writer; + int match = 0; ++ int rc; + + assert( dlrDocid(pLeft)==dlrDocid(pRight) ); + assert( pOut->iType!=DL_POSITIONS_OFFSETS ); + +- plrInit(&left, pLeft); +- plrInit(&right, pRight); ++ rc = plrInit(&left, pLeft); ++ if( rc!=SQLITE_OK ) return rc; ++ rc = plrInit(&right, pRight); ++ if( rc!=SQLITE_OK ){ ++ plrDestroy(&left); ++ return rc; ++ } + + while( !plrAtEnd(&left) && !plrAtEnd(&right) ){ + if( plrColumn(&left)<plrColumn(&right) ){ +- plrStep(&left); ++ rc = plrStep(&left); ++ if( rc!=SQLITE_OK ) break; + }else if( plrColumn(&left)>plrColumn(&right) ){ +- plrStep(&right); ++ rc = plrStep(&right); ++ if( rc!=SQLITE_OK ) break; + }else if( plrPosition(&left)>=plrPosition(&right) ){ +- plrStep(&right); ++ rc = plrStep(&right); ++ if( rc!=SQLITE_OK ) break; + }else{ + if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){ + if( !match ){ +@@ -1429,9 +1534,11 @@ + }else{ + plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0); + } +- plrStep(&right); ++ rc = plrStep(&right); ++ if( rc!=SQLITE_OK ) break; + }else{ +- plrStep(&left); ++ rc = plrStep(&left); ++ if( rc!=SQLITE_OK ) break; + } + } + } +@@ -1443,6 +1550,7 @@ + + plrDestroy(&left); + plrDestroy(&right); ++ return rc; + } + + /* +@@ -1496,7 +1604,7 @@ + ** iType controls the type of data written to pOut. If iType is + ** DL_POSITIONS, the positions are those from pRight. + */ +-static void docListPhraseMerge( ++static int docListPhraseMerge( + const char *pLeft, int nLeft, + const char *pRight, int nRight, + int nNear, /* 0 for a phrase merge, non-zero for a NEAR merge */ +@@ -1506,43 +1614,63 @@ + ){ + DLReader left, right; + DLWriter writer; ++ int rc; + +- if( nLeft==0 || nRight==0 ) return; ++ /* These two buffers are used in the 'while', but are declared here ++ ** to simplify error-handling. ++ */ ++ DataBuffer one = {0, 0, 0}; ++ DataBuffer two = {0, 0, 0}; + ++ if( nLeft==0 || nRight==0 ) return SQLITE_OK; ++ + assert( iType!=DL_POSITIONS_OFFSETS ); + +- dlrInit(&left, DL_POSITIONS, pLeft, nLeft); +- dlrInit(&right, DL_POSITIONS, pRight, nRight); ++ rc = dlrInit(&left, DL_POSITIONS, pLeft, nLeft); ++ if( rc!=SQLITE_OK ) return rc; ++ rc = dlrInit(&right, DL_POSITIONS, pRight, nRight); ++ if( rc!=SQLITE_OK ){ ++ dlrDestroy(&left); ++ return rc; ++ } + dlwInit(&writer, iType, pOut); + + while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){ + if( dlrDocid(&left)<dlrDocid(&right) ){ +- dlrStep(&left); ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK ) goto err; + }else if( dlrDocid(&right)<dlrDocid(&left) ){ +- dlrStep(&right); ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) goto err; + }else{ + if( nNear==0 ){ +- posListPhraseMerge(&left, &right, 0, 0, &writer); ++ rc = posListPhraseMerge(&left, &right, 0, 0, &writer); ++ if( rc!=SQLITE_OK ) goto err; + }else{ + /* This case occurs when two terms (simple terms or phrases) are + * connected by a NEAR operator, span (nNear+1). i.e. + * + * '"terrible company" NEAR widget' + */ +- DataBuffer one = {0, 0, 0}; +- DataBuffer two = {0, 0, 0}; +- + DLWriter dlwriter2; +- DLReader dr1 = {0, 0, 0, 0, 0}; ++ DLReader dr1 = {0, 0, 0, 0, 0}; + DLReader dr2 = {0, 0, 0, 0, 0}; + + dlwInit(&dlwriter2, iType, &one); +- posListPhraseMerge(&right, &left, nNear-3+nPhrase, 1, &dlwriter2); ++ rc = posListPhraseMerge(&right, &left, nNear-3+nPhrase, 1, &dlwriter2); ++ if( rc!=SQLITE_OK ) goto err; + dlwInit(&dlwriter2, iType, &two); +- posListPhraseMerge(&left, &right, nNear-1, 0, &dlwriter2); ++ rc = posListPhraseMerge(&left, &right, nNear-1, 0, &dlwriter2); ++ if( rc!=SQLITE_OK ) goto err; + +- if( one.nData) dlrInit(&dr1, iType, one.pData, one.nData); +- if( two.nData) dlrInit(&dr2, iType, two.pData, two.nData); ++ if( one.nData){ ++ rc = dlrInit(&dr1, iType, one.pData, one.nData); ++ if( rc!=SQLITE_OK ) goto err; ++ } ++ if( two.nData){ ++ rc = dlrInit(&dr2, iType, two.pData, two.nData); ++ if( rc!=SQLITE_OK ) goto err; ++ } + + if( !dlrAtEnd(&dr1) || !dlrAtEnd(&dr2) ){ + PLReader pr1 = {0}; +@@ -1551,161 +1679,211 @@ + PLWriter plwriter; + plwInit(&plwriter, &writer, dlrDocid(dlrAtEnd(&dr1)?&dr2:&dr1)); + +- if( one.nData ) plrInit(&pr1, &dr1); +- if( two.nData ) plrInit(&pr2, &dr2); ++ if( one.nData ){ ++ rc = plrInit(&pr1, &dr1); ++ if( rc!=SQLITE_OK ) goto err; ++ } ++ if( two.nData ){ ++ rc = plrInit(&pr2, &dr2); ++ if( rc!=SQLITE_OK ) goto err; ++ } + while( !plrAtEnd(&pr1) || !plrAtEnd(&pr2) ){ + int iCompare = plrCompare(&pr1, &pr2); + switch( iCompare ){ + case -1: + plwCopy(&plwriter, &pr1); +- plrStep(&pr1); ++ rc = plrStep(&pr1); ++ if( rc!=SQLITE_OK ) goto err; + break; + case 1: + plwCopy(&plwriter, &pr2); +- plrStep(&pr2); ++ rc = plrStep(&pr2); ++ if( rc!=SQLITE_OK ) goto err; + break; + case 0: + plwCopy(&plwriter, &pr1); +- plrStep(&pr1); +- plrStep(&pr2); ++ rc = plrStep(&pr1); ++ if( rc!=SQLITE_OK ) goto err; ++ rc = plrStep(&pr2); ++ if( rc!=SQLITE_OK ) goto err; + break; + } + } + plwTerminate(&plwriter); + } +- dataBufferDestroy(&one); +- dataBufferDestroy(&two); ++ dataBufferReset(&one); ++ dataBufferReset(&two); + } +- dlrStep(&left); +- dlrStep(&right); ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK ) goto err; ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) goto err; + } + } + ++err: ++ dataBufferDestroy(&one); ++ dataBufferDestroy(&two); + dlrDestroy(&left); + dlrDestroy(&right); + dlwDestroy(&writer); ++ return rc; + } + + /* We have two DL_DOCIDS doclists: pLeft and pRight. + ** Write the intersection of these two doclists into pOut as a + ** DL_DOCIDS doclist. + */ +-static void docListAndMerge( ++static int docListAndMerge( + const char *pLeft, int nLeft, + const char *pRight, int nRight, + DataBuffer *pOut /* Write the combined doclist here */ + ){ + DLReader left, right; + DLWriter writer; ++ int rc; + +- if( nLeft==0 || nRight==0 ) return; ++ if( nLeft==0 || nRight==0 ) return SQLITE_OK; + +- dlrInit(&left, DL_DOCIDS, pLeft, nLeft); +- dlrInit(&right, DL_DOCIDS, pRight, nRight); ++ rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft); ++ if( rc!=SQLITE_OK ) return rc; ++ rc = dlrInit(&right, DL_DOCIDS, pRight, nRight); ++ if( rc!=SQLITE_OK ){ ++ dlrDestroy(&left); ++ return rc; ++ } + dlwInit(&writer, DL_DOCIDS, pOut); + + while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){ + if( dlrDocid(&left)<dlrDocid(&right) ){ +- dlrStep(&left); ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK ) break; + }else if( dlrDocid(&right)<dlrDocid(&left) ){ +- dlrStep(&right); ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) break; + }else{ + dlwAdd(&writer, dlrDocid(&left)); +- dlrStep(&left); +- dlrStep(&right); ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK ) break; ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) break; + } + } + + dlrDestroy(&left); + dlrDestroy(&right); + dlwDestroy(&writer); ++ return rc; + } + + /* We have two DL_DOCIDS doclists: pLeft and pRight. + ** Write the union of these two doclists into pOut as a + ** DL_DOCIDS doclist. + */ +-static void docListOrMerge( ++static int docListOrMerge( + const char *pLeft, int nLeft, + const char *pRight, int nRight, + DataBuffer *pOut /* Write the combined doclist here */ + ){ + DLReader left, right; + DLWriter writer; ++ int rc; + + if( nLeft==0 ){ + if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight); +- return; ++ return SQLITE_OK; + } + if( nRight==0 ){ + dataBufferAppend(pOut, pLeft, nLeft); +- return; ++ return SQLITE_OK; + } + +- dlrInit(&left, DL_DOCIDS, pLeft, nLeft); +- dlrInit(&right, DL_DOCIDS, pRight, nRight); ++ rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft); ++ if( rc!=SQLITE_OK ) return rc; ++ rc = dlrInit(&right, DL_DOCIDS, pRight, nRight); ++ if( rc!=SQLITE_OK ){ ++ dlrDestroy(&left); ++ return rc; ++ } + dlwInit(&writer, DL_DOCIDS, pOut); + + while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){ + if( dlrAtEnd(&right) ){ + dlwAdd(&writer, dlrDocid(&left)); +- dlrStep(&left); ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK ) break; + }else if( dlrAtEnd(&left) ){ + dlwAdd(&writer, dlrDocid(&right)); +- dlrStep(&right); ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) break; + }else if( dlrDocid(&left)<dlrDocid(&right) ){ + dlwAdd(&writer, dlrDocid(&left)); +- dlrStep(&left); ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK ) break; + }else if( dlrDocid(&right)<dlrDocid(&left) ){ + dlwAdd(&writer, dlrDocid(&right)); +- dlrStep(&right); ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) break; + }else{ + dlwAdd(&writer, dlrDocid(&left)); +- dlrStep(&left); +- dlrStep(&right); ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK ) break; ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) break; + } + } + + dlrDestroy(&left); + dlrDestroy(&right); + dlwDestroy(&writer); ++ return rc; + } + + /* We have two DL_DOCIDS doclists: pLeft and pRight. + ** Write into pOut as DL_DOCIDS doclist containing all documents that + ** occur in pLeft but not in pRight. + */ +-static void docListExceptMerge( ++static int docListExceptMerge( + const char *pLeft, int nLeft, + const char *pRight, int nRight, + DataBuffer *pOut /* Write the combined doclist here */ + ){ + DLReader left, right; + DLWriter writer; ++ int rc; + +- if( nLeft==0 ) return; ++ if( nLeft==0 ) return SQLITE_OK; + if( nRight==0 ){ + dataBufferAppend(pOut, pLeft, nLeft); +- return; ++ return SQLITE_OK; + } + +- dlrInit(&left, DL_DOCIDS, pLeft, nLeft); +- dlrInit(&right, DL_DOCIDS, pRight, nRight); ++ rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft); ++ if( rc!=SQLITE_OK ) return rc; ++ rc = dlrInit(&right, DL_DOCIDS, pRight, nRight); ++ if( rc!=SQLITE_OK ){ ++ dlrDestroy(&left); ++ return rc; ++ } + dlwInit(&writer, DL_DOCIDS, pOut); + + while( !dlrAtEnd(&left) ){ + while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){ +- dlrStep(&right); ++ rc = dlrStep(&right); ++ if( rc!=SQLITE_OK ) goto err; + } + if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){ + dlwAdd(&writer, dlrDocid(&left)); + } +- dlrStep(&left); ++ rc = dlrStep(&left); ++ if( rc!=SQLITE_OK ) break; + } + ++err: + dlrDestroy(&left); + dlrDestroy(&right); + dlwDestroy(&writer); ++ return rc; + } + + static char *string_dup_n(const char *s, int n){ +@@ -1858,7 +2036,7 @@ + /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?", + /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)", + /* SEGDIR_SELECT_LEVEL */ +- "select start_block, leaves_end_block, root from %_segdir " ++ "select start_block, leaves_end_block, root, idx from %_segdir " + " where level = ? order by idx", + /* SEGDIR_SPAN */ + "select min(start_block), max(end_block) from %_segdir " +@@ -3680,16 +3858,19 @@ + return SQLITE_OK; + } + rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader)); +- dlrStep(&c->reader); + if( rc!=SQLITE_OK ) return rc; ++ rc = dlrStep(&c->reader); ++ if( rc!=SQLITE_OK ) return rc; + /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ + rc = sqlite3_step(c->pStmt); + if( rc==SQLITE_ROW ){ /* the case we expect */ + c->eof = 0; + return SQLITE_OK; + } +- /* an error occurred; abort */ +- return rc==SQLITE_DONE ? SQLITE_ERROR : rc; ++ /* Corrupt if the index refers to missing document. */ ++ if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT; ++ ++ return rc; + } + } + +@@ -3739,11 +3920,12 @@ + if( ii==(pPhrase->nToken-1) ){ + eType = eListType; + } +- docListPhraseMerge( ++ rc = docListPhraseMerge( + res.pData, res.nData, tmp.pData, tmp.nData, 0, 0, eType, pResult + ); + dataBufferDestroy(&res); + dataBufferDestroy(&tmp); ++ if( rc!= SQLITE_OK ) return rc; + } + } + } +@@ -3798,21 +3980,21 @@ + assert( pExpr->pRight->eType==FTSQUERY_PHRASE ); + assert( pLeft->eType==FTSQUERY_PHRASE ); + nToken = pLeft->pPhrase->nToken + pExpr->pRight->pPhrase->nToken; +- docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, ++ rc = docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, + pExpr->nNear+1, nToken, eType, pRes + ); + break; + } + case FTSQUERY_NOT: { +- docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes); ++ rc = docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes); + break; + } + case FTSQUERY_AND: { +- docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); ++ rc = docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); + break; + } + case FTSQUERY_OR: { +- docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); ++ rc = docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); + break; + } + } +@@ -4469,22 +4651,19 @@ + SCRAMBLE(pReader); + } + +-/* TODO(shess) The assertions are great, but what if we're in NDEBUG +-** and the blob is empty or otherwise contains suspect data? +-*/ +-static void interiorReaderInit(const char *pData, int nData, +- InteriorReader *pReader){ ++static int interiorReaderInit(const char *pData, int nData, ++ InteriorReader *pReader){ + int n, nTerm; + +- /* Require at least the leading flag byte */ ++ /* These conditions are checked and met by the callers. */ + assert( nData>0 ); + assert( pData[0]!='\0' ); + + CLEAR(pReader); + + /* Decode the base blockid, and set the cursor to the first term. */ +- n = fts3GetVarint(pData+1, &pReader->iBlockid); +- assert( 1+n<=nData ); ++ n = fts3GetVarintSafe(pData+1, &pReader->iBlockid, nData-1); ++ if( !n ) return SQLITE_CORRUPT_BKPT; + pReader->pData = pData+1+n; + pReader->nData = nData-(1+n); + +@@ -4495,17 +4674,18 @@ + if( pReader->nData==0 ){ + dataBufferInit(&pReader->term, 0); + }else{ +- n = fts3GetVarint32(pReader->pData, &nTerm); ++ n = fts3GetVarint32Safe(pReader->pData, &nTerm, pReader->nData); ++ if( !n || nTerm<0 || nTerm>pReader->nData-n) return SQLITE_CORRUPT_BKPT; + dataBufferInit(&pReader->term, nTerm); + dataBufferReplace(&pReader->term, pReader->pData+n, nTerm); +- assert( n+nTerm<=pReader->nData ); + pReader->pData += n+nTerm; + pReader->nData -= n+nTerm; + } ++ return SQLITE_OK; + } + + static int interiorReaderAtEnd(InteriorReader *pReader){ +- return pReader->term.nData==0; ++ return pReader->term.nData<=0; + } + + static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){ +@@ -4522,7 +4702,7 @@ + } + + /* Step forward to the next term in the node. */ +-static void interiorReaderStep(InteriorReader *pReader){ ++static int interiorReaderStep(InteriorReader *pReader){ + assert( !interiorReaderAtEnd(pReader) ); + + /* If the last term has been read, signal eof, else construct the +@@ -4533,18 +4713,26 @@ + }else{ + int n, nPrefix, nSuffix; + +- n = fts3GetVarint32(pReader->pData, &nPrefix); +- n += fts3GetVarint32(pReader->pData+n, &nSuffix); ++ n = fts3GetVarint32Safe(pReader->pData, &nPrefix, pReader->nData); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ pReader->nData -= n; ++ pReader->pData += n; ++ n = fts3GetVarint32Safe(pReader->pData, &nSuffix, pReader->nData); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ pReader->nData -= n; ++ pReader->pData += n; ++ if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT; ++ if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT; + + /* Truncate the current term and append suffix data. */ + pReader->term.nData = nPrefix; +- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); ++ dataBufferAppend(&pReader->term, pReader->pData, nSuffix); + +- assert( n+nSuffix<=pReader->nData ); +- pReader->pData += n+nSuffix; +- pReader->nData -= n+nSuffix; ++ pReader->pData += nSuffix; ++ pReader->nData -= nSuffix; + } + pReader->iBlockid++; ++ return SQLITE_OK; + } + + /* Compare the current term to pTerm[nTerm], returning strcmp-style +@@ -4916,7 +5104,8 @@ + n = fts3PutVarint(c, nData); + dataBufferAppend(&pWriter->data, c, n); + +- docListMerge(&pWriter->data, pReaders, nReaders); ++ rc = docListMerge(&pWriter->data, pReaders, nReaders); ++ if( rc!=SQLITE_OK ) return rc; + ASSERT_VALID_DOCLIST(DL_DEFAULT, + pWriter->data.pData+iDoclistData+n, + pWriter->data.nData-iDoclistData-n, NULL); +@@ -5026,7 +5215,8 @@ + int rc; + DLReader reader; + +- dlrInit(&reader, DL_DEFAULT, pData, nData); ++ rc = dlrInit(&reader, DL_DEFAULT, pData, nData); ++ if( rc!=SQLITE_OK ) return rc; + rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1); + dlrDestroy(&reader); + +@@ -5071,38 +5261,41 @@ + static const char *leafReaderData(LeafReader *pReader){ + int n, nData; + assert( pReader->term.nData>0 ); +- n = fts3GetVarint32(pReader->pData, &nData); ++ n = fts3GetVarint32Safe(pReader->pData, &nData, pReader->nData); ++ if( !n || nData>pReader->nData-n ) return NULL; + return pReader->pData+n; + } + +-static void leafReaderInit(const char *pData, int nData, +- LeafReader *pReader){ ++static int leafReaderInit(const char *pData, int nData, ++ LeafReader *pReader){ + int nTerm, n; + ++ /* All callers check this precondition. */ + assert( nData>0 ); + assert( pData[0]=='\0' ); + + CLEAR(pReader); + + /* Read the first term, skipping the header byte. */ +- n = fts3GetVarint32(pData+1, &nTerm); ++ n = fts3GetVarint32Safe(pData+1, &nTerm, nData-1); ++ if( !n || nTerm<0 || nTerm>nData-1-n ) return SQLITE_CORRUPT_BKPT; + dataBufferInit(&pReader->term, nTerm); + dataBufferReplace(&pReader->term, pData+1+n, nTerm); + + /* Position after the first term. */ +- assert( 1+n+nTerm<nData ); + pReader->pData = pData+1+n+nTerm; + pReader->nData = nData-1-n-nTerm; ++ return SQLITE_OK; + } + + /* Step the reader forward to the next term. */ +-static void leafReaderStep(LeafReader *pReader){ ++static int leafReaderStep(LeafReader *pReader){ + int n, nData, nPrefix, nSuffix; + assert( !leafReaderAtEnd(pReader) ); + + /* Skip previous entry's data block. */ +- n = fts3GetVarint32(pReader->pData, &nData); +- assert( n+nData<=pReader->nData ); ++ n = fts3GetVarint32Safe(pReader->pData, &nData, pReader->nData); ++ if( !n || nData<0 || nData>pReader->nData-n ) return SQLITE_CORRUPT_BKPT; + pReader->pData += n+nData; + pReader->nData -= n+nData; + +@@ -5110,15 +5303,23 @@ + /* Construct the new term using a prefix from the old term plus a + ** suffix from the leaf data. + */ +- n = fts3GetVarint32(pReader->pData, &nPrefix); +- n += fts3GetVarint32(pReader->pData+n, &nSuffix); +- assert( n+nSuffix<pReader->nData ); ++ n = fts3GetVarint32Safe(pReader->pData, &nPrefix, pReader->nData); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ pReader->nData -= n; ++ pReader->pData += n; ++ n = fts3GetVarint32Safe(pReader->pData, &nSuffix, pReader->nData); ++ if( !n ) return SQLITE_CORRUPT_BKPT; ++ pReader->nData -= n; ++ pReader->pData += n; ++ if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT; ++ if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT; + pReader->term.nData = nPrefix; +- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); ++ dataBufferAppend(&pReader->term, pReader->pData, nSuffix); + +- pReader->pData += n+nSuffix; +- pReader->nData -= n+nSuffix; ++ pReader->pData += nSuffix; ++ pReader->nData -= nSuffix; + } ++ return SQLITE_OK; + } + + /* strcmp-style comparison of pReader's current term against pTerm. +@@ -5222,32 +5423,67 @@ + + dataBufferInit(&pReader->rootData, 0); + if( iStartBlockid==0 ){ ++ int rc; ++ /* Corrupt if this can't be a leaf node. */ ++ if( pRootData==NULL || nRootData<1 || pRootData[0]!='\0' ){ ++ return SQLITE_CORRUPT_BKPT; ++ } + /* Entire leaf level fit in root data. */ + dataBufferReplace(&pReader->rootData, pRootData, nRootData); +- leafReaderInit(pReader->rootData.pData, pReader->rootData.nData, +- &pReader->leafReader); ++ rc = leafReaderInit(pReader->rootData.pData, pReader->rootData.nData, ++ &pReader->leafReader); ++ if( rc!=SQLITE_OK ){ ++ dataBufferDestroy(&pReader->rootData); ++ return rc; ++ } + }else{ + sqlite3_stmt *s; + int rc = sql_get_leaf_statement(v, idx, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_int64(s, 1, iStartBlockid); +- if( rc!=SQLITE_OK ) return rc; ++ if( rc!=SQLITE_OK ) goto err; + + rc = sqlite3_bind_int64(s, 2, iEndBlockid); +- if( rc!=SQLITE_OK ) return rc; ++ if( rc!=SQLITE_OK ) goto err; + + rc = sqlite3_step(s); ++ ++ /* Corrupt if interior node referenced missing leaf node. */ + if( rc==SQLITE_DONE ){ +- pReader->eof = 1; +- return SQLITE_OK; ++ rc = SQLITE_CORRUPT_BKPT; ++ goto err; + } +- if( rc!=SQLITE_ROW ) return rc; + ++ if( rc!=SQLITE_ROW ) goto err; ++ rc = SQLITE_OK; ++ ++ /* Corrupt if leaf data isn't a blob. */ ++ if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ }else{ ++ const char *pLeafData = sqlite3_column_blob(s, 0); ++ int nLeafData = sqlite3_column_bytes(s, 0); ++ ++ /* Corrupt if this can't be a leaf node. */ ++ if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ }else{ ++ rc = leafReaderInit(pLeafData, nLeafData, &pReader->leafReader); ++ } ++ } ++ ++ err: ++ if( rc!=SQLITE_OK ){ ++ if( idx==-1 ){ ++ sqlite3_finalize(s); ++ }else{ ++ sqlite3_reset(s); ++ } ++ return rc; ++ } ++ + pReader->pStmt = s; +- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0), +- sqlite3_column_bytes(pReader->pStmt, 0), +- &pReader->leafReader); + } + return SQLITE_OK; + } +@@ -5256,11 +5492,12 @@ + ** end of the current leaf, step forward to the next leaf block. + */ + static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){ ++ int rc; + assert( !leavesReaderAtEnd(pReader) ); +- leafReaderStep(&pReader->leafReader); ++ rc = leafReaderStep(&pReader->leafReader); ++ if( rc!=SQLITE_OK ) return rc; + + if( leafReaderAtEnd(&pReader->leafReader) ){ +- int rc; + if( pReader->rootData.pData ){ + pReader->eof = 1; + return SQLITE_OK; +@@ -5270,10 +5507,25 @@ + pReader->eof = 1; + return rc==SQLITE_DONE ? SQLITE_OK : rc; + } +- leafReaderDestroy(&pReader->leafReader); +- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0), +- sqlite3_column_bytes(pReader->pStmt, 0), +- &pReader->leafReader); ++ ++ /* Corrupt if leaf data isn't a blob. */ ++ if( sqlite3_column_type(pReader->pStmt, 0)!=SQLITE_BLOB ){ ++ return SQLITE_CORRUPT_BKPT; ++ }else{ ++ LeafReader tmp; ++ const char *pLeafData = sqlite3_column_blob(pReader->pStmt, 0); ++ int nLeafData = sqlite3_column_bytes(pReader->pStmt, 0); ++ ++ /* Corrupt if this can't be a leaf node. */ ++ if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){ ++ return SQLITE_CORRUPT_BKPT; ++ } ++ ++ rc = leafReaderInit(pLeafData, nLeafData, &tmp); ++ if( rc!=SQLITE_OK ) return rc; ++ leafReaderDestroy(&pReader->leafReader); ++ pReader->leafReader = tmp; ++ } + } + return SQLITE_OK; + } +@@ -5334,8 +5586,19 @@ + sqlite_int64 iEnd = sqlite3_column_int64(s, 1); + const char *pRootData = sqlite3_column_blob(s, 2); + int nRootData = sqlite3_column_bytes(s, 2); ++ sqlite_int64 iIndex = sqlite3_column_int64(s, 3); + +- assert( i<MERGE_COUNT ); ++ /* Corrupt if we get back different types than we stored. */ ++ /* Also corrupt if the index is not sequential starting at 0. */ ++ if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER || ++ sqlite3_column_type(s, 1)!=SQLITE_INTEGER || ++ sqlite3_column_type(s, 2)!=SQLITE_BLOB || ++ i!=iIndex || ++ i>=MERGE_COUNT ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ break; ++ } ++ + rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData, + &pReaders[i]); + if( rc!=SQLITE_OK ) break; +@@ -5346,6 +5609,7 @@ + while( i-->0 ){ + leavesReaderDestroy(&pReaders[i]); + } ++ sqlite3_reset(s); /* So we don't leave a lock. */ + return rc; + } + +@@ -5369,14 +5633,27 @@ + DLReader dlReaders[MERGE_COUNT]; + const char *pTerm = leavesReaderTerm(pReaders); + int i, nTerm = leavesReaderTermBytes(pReaders); ++ int rc; + + assert( nReaders<=MERGE_COUNT ); + + for(i=0; i<nReaders; i++){ +- dlrInit(&dlReaders[i], DL_DEFAULT, +- leavesReaderData(pReaders+i), +- leavesReaderDataBytes(pReaders+i)); ++ const char *pData = leavesReaderData(pReaders+i); ++ if( pData==NULL ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ break; ++ } ++ rc = dlrInit(&dlReaders[i], DL_DEFAULT, ++ pData, ++ leavesReaderDataBytes(pReaders+i)); ++ if( rc!=SQLITE_OK ) break; + } ++ if( rc!=SQLITE_OK ){ ++ while( i-->0 ){ ++ dlrDestroy(&dlReaders[i]); ++ } ++ return rc; ++ } + + return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders); + } +@@ -5429,10 +5706,14 @@ + memset(&lrs, '\0', sizeof(lrs)); + rc = leavesReadersInit(v, iLevel, lrs, &i); + if( rc!=SQLITE_OK ) return rc; +- assert( i==MERGE_COUNT ); + + leafWriterInit(iLevel+1, idx, &writer); + ++ if( i!=MERGE_COUNT ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ goto err; ++ } ++ + /* Since leavesReaderReorder() pushes readers at eof to the end, + ** when the first reader is empty, all will be empty. + */ +@@ -5475,12 +5756,14 @@ + } + + /* Accumulate the union of *acc and *pData into *acc. */ +-static void docListAccumulateUnion(DataBuffer *acc, +- const char *pData, int nData) { ++static int docListAccumulateUnion(DataBuffer *acc, ++ const char *pData, int nData) { + DataBuffer tmp = *acc; ++ int rc; + dataBufferInit(acc, tmp.nData+nData); +- docListUnion(tmp.pData, tmp.nData, pData, nData, acc); ++ rc = docListUnion(tmp.pData, tmp.nData, pData, nData, acc); + dataBufferDestroy(&tmp); ++ return rc; + } + + /* TODO(shess) It might be interesting to explore different merge +@@ -5522,8 +5805,13 @@ + int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix); + if( c>0 ) break; /* Past any possible matches. */ + if( c==0 ){ ++ int iBuffer, nData; + const char *pData = leavesReaderData(pReader); +- int iBuffer, nData = leavesReaderDataBytes(pReader); ++ if( pData==NULL ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ break; ++ } ++ nData = leavesReaderDataBytes(pReader); + + /* Find the first empty buffer. */ + for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){ +@@ -5569,11 +5857,13 @@ + ** with pData/nData. + */ + dataBufferSwap(p, pAcc); +- docListAccumulateUnion(pAcc, pData, nData); ++ rc = docListAccumulateUnion(pAcc, pData, nData); ++ if( rc!=SQLITE_OK ) goto err; + + /* Accumulate remaining doclists into pAcc. */ + for(++p; p<pAcc; ++p){ +- docListAccumulateUnion(pAcc, p->pData, p->nData); ++ rc = docListAccumulateUnion(pAcc, p->pData, p->nData); ++ if( rc!=SQLITE_OK ) goto err; + + /* dataBufferReset() could allow a large doclist to blow up + ** our memory requirements. +@@ -5598,13 +5888,15 @@ + if( out->nData==0 ){ + dataBufferSwap(out, &(pBuffers[iBuffer])); + }else{ +- docListAccumulateUnion(out, pBuffers[iBuffer].pData, +- pBuffers[iBuffer].nData); ++ rc = docListAccumulateUnion(out, pBuffers[iBuffer].pData, ++ pBuffers[iBuffer].nData); ++ if( rc!=SQLITE_OK ) break; + } + } + } + } + ++err: + while( nBuffers-- ){ + dataBufferDestroy(&(pBuffers[nBuffers])); + } +@@ -5663,20 +5955,26 @@ + ** node. Consider whether breaking symmetry is worthwhile. I suspect + ** it is not worthwhile. + */ +-static void getChildrenContaining(const char *pData, int nData, +- const char *pTerm, int nTerm, int isPrefix, +- sqlite_int64 *piStartChild, +- sqlite_int64 *piEndChild){ ++static int getChildrenContaining(const char *pData, int nData, ++ const char *pTerm, int nTerm, int isPrefix, ++ sqlite_int64 *piStartChild, ++ sqlite_int64 *piEndChild){ + InteriorReader reader; ++ int rc; + + assert( nData>1 ); + assert( *pData!='\0' ); +- interiorReaderInit(pData, nData, &reader); ++ rc = interiorReaderInit(pData, nData, &reader); ++ if( rc!=SQLITE_OK ) return rc; + + /* Scan for the first child which could contain pTerm/nTerm. */ + while( !interiorReaderAtEnd(&reader) ){ + if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break; +- interiorReaderStep(&reader); ++ rc = interiorReaderStep(&reader); ++ if( rc!=SQLITE_OK ){ ++ interiorReaderDestroy(&reader); ++ return rc; ++ } + } + *piStartChild = interiorReaderCurrentBlockid(&reader); + +@@ -5686,7 +5984,11 @@ + */ + while( !interiorReaderAtEnd(&reader) ){ + if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break; +- interiorReaderStep(&reader); ++ rc = interiorReaderStep(&reader); ++ if( rc!=SQLITE_OK ){ ++ interiorReaderDestroy(&reader); ++ return rc; ++ } + } + *piEndChild = interiorReaderCurrentBlockid(&reader); + +@@ -5695,6 +5997,7 @@ + /* Children must ascend, and if !prefix, both must be the same. */ + assert( *piEndChild>=*piStartChild ); + assert( isPrefix || *piStartChild==*piEndChild ); ++ return rc; + } + + /* Read block at iBlockid and pass it with other params to +@@ -5722,12 +6025,32 @@ + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_step(s); +- if( rc==SQLITE_DONE ) return SQLITE_ERROR; ++ /* Corrupt if interior node references missing child node. */ ++ if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT; + if( rc!=SQLITE_ROW ) return rc; + +- getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0), +- pTerm, nTerm, isPrefix, piStartChild, piEndChild); ++ /* Corrupt if child node isn't a blob. */ ++ if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){ ++ sqlite3_reset(s); /* So we don't leave a lock. */ ++ return SQLITE_CORRUPT_BKPT; ++ }else{ ++ const char *pData = sqlite3_column_blob(s, 0); ++ int nData = sqlite3_column_bytes(s, 0); + ++ /* Corrupt if child is not a valid interior node. */ ++ if( pData==NULL || nData<1 || pData[0]=='\0' ){ ++ sqlite3_reset(s); /* So we don't leave a lock. */ ++ return SQLITE_CORRUPT_BKPT; ++ } ++ ++ rc = getChildrenContaining(pData, nData, pTerm, nTerm, ++ isPrefix, piStartChild, piEndChild); ++ if( rc!=SQLITE_OK ){ ++ sqlite3_reset(s); ++ return rc; ++ } ++ } ++ + /* We expect only one row. We must execute another sqlite3_step() + * to complete the iteration; otherwise the table will remain + * locked. */ +@@ -5756,8 +6079,9 @@ + /* Process pData as an interior node, then loop down the tree + ** until we find the set of leaf nodes to scan for the term. + */ +- getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix, +- &iStartChild, &iEndChild); ++ rc = getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix, ++ &iStartChild, &iEndChild); ++ if( rc!=SQLITE_OK ) return rc; + while( iStartChild>iLeavesEnd ){ + sqlite_int64 iNextStart, iNextEnd; + rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix, +@@ -5809,7 +6133,8 @@ + DataBuffer result; + int rc; + +- assert( nData>1 ); ++ /* Corrupt if segment root can't be valid. */ ++ if( pData==NULL || nData<1 ) return SQLITE_CORRUPT_BKPT; + + /* This code should never be called with buffered updates. */ + assert( v->nPendingData<0 ); +@@ -5826,16 +6151,21 @@ + DataBuffer merged; + DLReader readers[2]; + +- dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData); +- dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData); +- dataBufferInit(&merged, out->nData+result.nData); +- docListMerge(&merged, readers, 2); +- dataBufferDestroy(out); +- *out = merged; +- dlrDestroy(&readers[0]); +- dlrDestroy(&readers[1]); ++ rc = dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData); ++ if( rc==SQLITE_OK ){ ++ rc = dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData); ++ if( rc==SQLITE_OK ){ ++ dataBufferInit(&merged, out->nData+result.nData); ++ rc = docListMerge(&merged, readers, 2); ++ dataBufferDestroy(out); ++ *out = merged; ++ dlrDestroy(&readers[1]); ++ } ++ dlrDestroy(&readers[0]); ++ } + } + } ++ + dataBufferDestroy(&result); + return rc; + } +@@ -5869,11 +6199,20 @@ + const char *pData = sqlite3_column_blob(s, 2); + const int nData = sqlite3_column_bytes(s, 2); + const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); ++ ++ /* Corrupt if we get back different types than we stored. */ ++ if( sqlite3_column_type(s, 1)!=SQLITE_INTEGER || ++ sqlite3_column_type(s, 2)!=SQLITE_BLOB ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ goto err; ++ } ++ + rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix, + &doclist); + if( rc!=SQLITE_OK ) goto err; + } + if( rc==SQLITE_DONE ){ ++ rc = SQLITE_OK; + if( doclist.nData!=0 ){ + /* TODO(shess) The old term_select_all() code applied the column + ** restrict as we merged segments, leading to smaller buffers. +@@ -5881,13 +6220,13 @@ + ** system is checked in. + */ + if( iColumn==v->nColumn) iColumn = -1; +- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, +- iColumn, iType, out); ++ rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, ++ iColumn, iType, out); + } +- rc = SQLITE_OK; + } + + err: ++ sqlite3_reset(s); /* So we don't leave a lock. */ + dataBufferDestroy(&doclist); + return rc; + } +@@ -6250,6 +6589,7 @@ + LeafWriter *pWriter){ + int i, rc = SQLITE_OK; + DataBuffer doclist, merged, tmp; ++ const char *pData; + + /* Order the readers. */ + i = nReaders; +@@ -6270,14 +6610,20 @@ + if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break; + } + ++ pData = optLeavesReaderData(&readers[0]); ++ if( pData==NULL ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ break; ++ } ++ + /* Special-case for no merge. */ + if( i==1 ){ + /* Trim deletions from the doclist. */ + dataBufferReset(&merged); +- docListTrim(DL_DEFAULT, +- optLeavesReaderData(&readers[0]), +- optLeavesReaderDataBytes(&readers[0]), +- -1, DL_DEFAULT, &merged); ++ rc = docListTrim(DL_DEFAULT, pData, ++ optLeavesReaderDataBytes(&readers[0]), ++ -1, DL_DEFAULT, &merged); ++ if( rc!=SQLITE_OK ) break; + }else{ + DLReader dlReaders[MERGE_COUNT]; + int iReader, nReaders; +@@ -6285,9 +6631,10 @@ + /* Prime the pipeline with the first reader's doclist. After + ** one pass index 0 will reference the accumulated doclist. + */ +- dlrInit(&dlReaders[0], DL_DEFAULT, +- optLeavesReaderData(&readers[0]), +- optLeavesReaderDataBytes(&readers[0])); ++ rc = dlrInit(&dlReaders[0], DL_DEFAULT, ++ pData, ++ optLeavesReaderDataBytes(&readers[0])); ++ if( rc!=SQLITE_OK ) break; + iReader = 1; + + assert( iReader<i ); /* Must execute the loop at least once. */ +@@ -6295,24 +6642,34 @@ + /* Merge 16 inputs per pass. */ + for( nReaders=1; iReader<i && nReaders<MERGE_COUNT; + iReader++, nReaders++ ){ +- dlrInit(&dlReaders[nReaders], DL_DEFAULT, +- optLeavesReaderData(&readers[iReader]), +- optLeavesReaderDataBytes(&readers[iReader])); ++ pData = optLeavesReaderData(&readers[iReader]); ++ if( pData==NULL ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ break; ++ } ++ rc = dlrInit(&dlReaders[nReaders], DL_DEFAULT, pData, ++ optLeavesReaderDataBytes(&readers[iReader])); ++ if( rc!=SQLITE_OK ) break; + } + + /* Merge doclists and swap result into accumulator. */ +- dataBufferReset(&merged); +- docListMerge(&merged, dlReaders, nReaders); +- tmp = merged; +- merged = doclist; +- doclist = tmp; ++ if( rc==SQLITE_OK ){ ++ dataBufferReset(&merged); ++ rc = docListMerge(&merged, dlReaders, nReaders); ++ tmp = merged; ++ merged = doclist; ++ doclist = tmp; ++ } + + while( nReaders-- > 0 ){ + dlrDestroy(&dlReaders[nReaders]); + } + ++ if( rc!=SQLITE_OK ) goto err; ++ + /* Accumulated doclist to reader 0 for next pass. */ +- dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData); ++ rc = dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData); ++ if( rc!=SQLITE_OK ) goto err; + } + + /* Destroy reader that was left in the pipeline. */ +@@ -6320,8 +6677,9 @@ + + /* Trim deletions from the doclist. */ + dataBufferReset(&merged); +- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, +- -1, DL_DEFAULT, &merged); ++ rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, ++ -1, DL_DEFAULT, &merged); ++ if( rc!=SQLITE_OK ) goto err; + } + + /* Only pass doclists with hits (skip if all hits deleted). */ +@@ -6401,6 +6759,14 @@ + const char *pRootData = sqlite3_column_blob(s, 2); + int nRootData = sqlite3_column_bytes(s, 2); + ++ /* Corrupt if we get back different types than we stored. */ ++ if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER || ++ sqlite3_column_type(s, 1)!=SQLITE_INTEGER || ++ sqlite3_column_type(s, 2)!=SQLITE_BLOB ){ ++ rc = SQLITE_CORRUPT_BKPT; ++ break; ++ } ++ + assert( i<nReaders ); + rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData, + &readers[i].reader); +@@ -6414,6 +6780,8 @@ + if( rc==SQLITE_DONE ){ + assert( i==nReaders ); + rc = optimizeInternal(v, readers, nReaders, &writer); ++ }else{ ++ sqlite3_reset(s); /* So we don't leave a lock. */ + } + + while( i-- > 0 ){ +@@ -6477,9 +6845,18 @@ + const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1); + const char *pRootData = sqlite3_column_blob(s, 2); + const int nRootData = sqlite3_column_bytes(s, 2); ++ int rc; + LeavesReader reader; +- int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, +- pRootData, nRootData, &reader); ++ ++ /* Corrupt if we get back different types than we stored. */ ++ if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER || ++ sqlite3_column_type(s, 1)!=SQLITE_INTEGER || ++ sqlite3_column_type(s, 2)!=SQLITE_BLOB ){ ++ return SQLITE_CORRUPT_BKPT; ++ } ++ ++ rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, ++ pRootData, nRootData, &reader); + if( rc!=SQLITE_OK ) return rc; + + while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){ +@@ -6641,16 +7018,19 @@ + const char *pData, int nData){ + DataBuffer dump; + DLReader dlReader; ++ int rc; + + assert( pData!=NULL && nData>0 ); + ++ rc = dlrInit(&dlReader, DL_DEFAULT, pData, nData); ++ if( rc!=SQLITE_OK ) return rc; + dataBufferInit(&dump, 0); +- dlrInit(&dlReader, DL_DEFAULT, pData, nData); +- for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){ ++ for( ; rc==SQLITE_OK && !dlrAtEnd(&dlReader); rc = dlrStep(&dlReader) ){ + char buf[256]; + PLReader plReader; + +- plrInit(&plReader, &dlReader); ++ rc = plrInit(&plReader, &dlReader); ++ if( rc!=SQLITE_OK ) break; + if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){ + sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader)); + dataBufferAppend(&dump, buf, strlen(buf)); +@@ -6661,7 +7041,8 @@ + dlrDocid(&dlReader), iColumn); + dataBufferAppend(&dump, buf, strlen(buf)); + +- for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){ ++ for( ; !plrAtEnd(&plReader); rc = plrStep(&plReader) ){ ++ if( rc!=SQLITE_OK ) break; + if( plrColumn(&plReader)!=iColumn ){ + iColumn = plrColumn(&plReader); + sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn); +@@ -6682,6 +7063,7 @@ + dataBufferAppend(&dump, buf, strlen(buf)); + } + plrDestroy(&plReader); ++ if( rc!= SQLITE_OK ) break; + + assert( dump.nData>0 ); + dump.nData--; /* Overwrite trailing space. */ +@@ -6690,6 +7072,10 @@ + } + } + dlrDestroy(&dlReader); ++ if( rc!=SQLITE_OK ){ ++ dataBufferDestroy(&dump); ++ return rc; ++ } + + assert( dump.nData>0 ); + dump.nData--; /* Overwrite trailing space. */ +@@ -6701,6 +7087,7 @@ + sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free); + dump.pData = NULL; + dump.nData = dump.nCapacity = 0; ++ return SQLITE_OK; + } + + /* Implements dump_doclist() for use in inspecting the fts3 index from +@@ -6987,7 +7374,11 @@ + ** module with sqlite. + */ + if( SQLITE_OK==rc ++#if CHROMIUM_FTS3_CHANGES && !SQLITE_TEST ++ /* fts3_tokenizer() disabled for security reasons. */ ++#else + && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) ++#endif + && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1)) +Index: ext/fts3/fts3_icu.c +=================================================================== +--- ext/fts3/fts3_icu.c (revision 48811) ++++ ext/fts3/fts3_icu.c (working copy) +@@ -198,7 +198,7 @@ while( iStart<iEnd ){ int iWhite = iStart; @@ -32,15 +1912,15 @@ index 85390d3..a75b14a 100644 if( u_isspace(c) ){ iStart = iWhite; }else{ -diff --git a/third_party/sqlite/src/ext/fts3/fts3_tokenizer.c b/third_party/sqlite/src/ext/fts3/fts3_tokenizer.c -index 54a91bf..1121b00 100644 ---- a/third_party/sqlite/src/ext/fts3/fts3_tokenizer.c -+++ b/third_party/sqlite/src/ext/fts3/fts3_tokenizer.c -@@ -32,6 +32,7 @@ - - #include "fts3Int.h" +Index: ext/fts3/fts3_tokenizer.c +=================================================================== +--- ext/fts3/fts3_tokenizer.c (revision 48811) ++++ ext/fts3/fts3_tokenizer.c (working copy) +@@ -33,6 +33,7 @@ + #include "fts3_hash.h" + #include "fts3_tokenizer.h" #include <assert.h> +#include <stddef.h> - #include <string.h> /* + ** Implementation of the SQL scalar function for accessing the underlying |