Update sqlite to 3.7.3.

BUG=none
TEST=none

Review URL: http://codereview.chromium.org/5626002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@68506 0039d316-1c4b-4281-b951-d872f2087c98

38 files changed, 7938 insertions, 7144 deletions

diff --git a/third_party/sqlite/src/ext/async/sqlite3async.c b/third_party/sqlite/src/ext/async/sqlite3async.c
index 11a4761..a351eaa 100644
--- a/third_party/sqlite/src/ext/async/sqlite3async.c
+++ b/third_party/sqlite/src/ext/async/sqlite3async.c
@@ -667,9 +667,9 @@ static int asyncRead(
 ){
   AsyncFileData *p = ((AsyncFile *)pFile)->pData;
   int rc = SQLITE_OK;
-  sqlite3_int64 filesize;
-  int nRead;
+  sqlite3_int64 filesize = 0;
   sqlite3_file *pBase = p->pBaseRead;
+  sqlite3_int64 iAmt64 = (sqlite3_int64)iAmt;
 
   /* Grab the write queue mutex for the duration of the call */
   async_mutex_enter(ASYNC_MUTEX_QUEUE);
@@ -683,13 +683,14 @@ static int asyncRead(
   }
 
   if( pBase->pMethods ){
+    sqlite3_int64 nRead;
     rc = pBase->pMethods->xFileSize(pBase, &filesize);
     if( rc!=SQLITE_OK ){
       goto asyncread_out;
     }
-    nRead = (int)MIN(filesize - iOffset, iAmt);
+    nRead = MIN(filesize - iOffset, iAmt64);
     if( nRead>0 ){
-      rc = pBase->pMethods->xRead(pBase, zOut, nRead, iOffset);
+      rc = pBase->pMethods->xRead(pBase, zOut, (int)nRead, iOffset);
       ASYNC_TRACE(("READ %s %d bytes at %d\n", p->zName, nRead, iOffset));
     }
   }
@@ -703,16 +704,24 @@ static int asyncRead(
         (pWrite->pFileData==p) ||
         (zName && pWrite->pFileData->zName==zName)
       )){
+        sqlite3_int64 nCopy;
+        sqlite3_int64 nByte64 = (sqlite3_int64)pWrite->nByte;
+
+        /* Set variable iBeginIn to the offset in buffer pWrite->zBuf[] from
+        ** which data should be copied. Set iBeginOut to the offset within
+        ** the output buffer to which data should be copied. If either of
+        ** these offsets is a negative number, set them to 0.
+        */
         sqlite3_int64 iBeginOut = (pWrite->iOffset-iOffset);
         sqlite3_int64 iBeginIn = -iBeginOut;
-        int nCopy;
-
         if( iBeginIn<0 ) iBeginIn = 0;
         if( iBeginOut<0 ) iBeginOut = 0;
-        nCopy = (int)MIN(pWrite->nByte-iBeginIn, iAmt-iBeginOut);
 
+        filesize = MAX(filesize, pWrite->iOffset+nByte64);
+
+        nCopy = MIN(nByte64-iBeginIn, iAmt64-iBeginOut);
         if( nCopy>0 ){
-          memcpy(&((char *)zOut)[iBeginOut], &pWrite->zBuf[iBeginIn], nCopy);
+          memcpy(&((char *)zOut)[iBeginOut], &pWrite->zBuf[iBeginIn], (size_t)nCopy);
           ASYNC_TRACE(("OVERREAD %d bytes at %d\n", nCopy, iBeginOut+iOffset));
         }
       }
@@ -721,6 +730,9 @@ static int asyncRead(
 
 asyncread_out:
   async_mutex_leave(ASYNC_MUTEX_QUEUE);
+  if( rc==SQLITE_OK && filesize<(iOffset+iAmt) ){
+    rc = SQLITE_IOERR_SHORT_READ;
+  }
   return rc;
 }
 
@@ -1131,7 +1143,6 @@ static int asyncOpen(
   async_mutex_leave(ASYNC_MUTEX_LOCK);
 
   if( rc==SQLITE_OK ){
-    incrOpenFileCount();
     pData->pLock = pLock;
   }
 
@@ -1148,7 +1159,10 @@ static int asyncOpen(
   }
   if( rc!=SQLITE_OK ){
     p->pMethod = 0;
+  }else{
+    incrOpenFileCount();
   }
+
   return rc;
 }
 
@@ -1224,8 +1238,8 @@ static int asyncFullPathname(
   */
   if( rc==SQLITE_OK ){
     int i, j;
-    int n = nPathOut;
     char *z = zPathOut;
+    int n = (int)strlen(z);
     while( n>1 && z[n-1]=='/' ){ n--; }
     for(i=j=0; i<n; i++){
       if( z[i]=='/' ){
diff --git a/third_party/sqlite/src/ext/fts2/README.tokenizers b/third_party/sqlite/src/ext/fts2/README.tokenizers
index 98d2021..98d2021 100755..100644
--- a/third_party/sqlite/src/ext/fts2/README.tokenizers
+++ b/third_party/sqlite/src/ext/fts2/README.tokenizers
diff --git a/third_party/sqlite/src/ext/fts2/fts2.c b/third_party/sqlite/src/ext/fts2/fts2.c
index ec4c108..5ec1265 100644
--- a/third_party/sqlite/src/ext/fts2/fts2.c
+++ b/third_party/sqlite/src/ext/fts2/fts2.c
@@ -320,8 +320,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include <ctype.h>
-
 #include "fts2.h"
 #include "fts2_hash.h"
 #include "fts2_tokenizer.h"
@@ -369,13 +367,13 @@ static int fts2Corrupt(void){
 */
 /* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
 static int safe_isspace(char c){
-  return (c&0x80)==0 ? isspace(c) : 0;
+  return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
 }
 static int safe_tolower(char c){
-  return (c>='A' && c<='Z') ? (c-'A'+'a') : c;
+  return (c>='A' && c<='Z') ? (c - 'A' + 'a') : c;
 }
 static int safe_isalnum(char c){
-  return (c&0x80)==0 ? isalnum(c) : 0;
+  return (c>='0' && c<='9') || (c>='A' && c<='Z') || (c>='a' && c<='z');
 }
 
 typedef enum DocListType {
diff --git a/third_party/sqlite/src/ext/fts2/fts2_porter.c b/third_party/sqlite/src/ext/fts2/fts2_porter.c
index 97a95c8..16620b9 100644
--- a/third_party/sqlite/src/ext/fts2/fts2_porter.c
+++ b/third_party/sqlite/src/ext/fts2/fts2_porter.c
@@ -29,7 +29,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include <ctype.h>
 
 #include "fts2_tokenizer.h"
 
diff --git a/third_party/sqlite/src/ext/fts2/fts2_tokenizer1.c b/third_party/sqlite/src/ext/fts2/fts2_tokenizer1.c
index 1f03cc4..7e13366 100644
--- a/third_party/sqlite/src/ext/fts2/fts2_tokenizer1.c
+++ b/third_party/sqlite/src/ext/fts2/fts2_tokenizer1.c
@@ -29,7 +29,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include <ctype.h>
 
 #include "fts2_tokenizer.h"
 
@@ -89,7 +88,8 @@ static int simpleCreate(
     /* Mark non-alphanumeric ASCII characters as delimiters */
     int i;
     for(i=1; i<0x80; i++){
-      t->delim[i] = !isalnum(i);
+      t->delim[i] = !((i>='0' && i<='9') || (i>='A' && i<='Z') ||
+                      (i>='a' && i<='z'));
     }
   }
 
@@ -191,7 +191,7 @@ static int simpleNext(
         ** case-insensitivity.
         */
         unsigned char ch = p[iStartOffset+i];
-        c->pToken[i] = (ch>='A' && ch<='Z') ? (ch-'A'+'a') : ch;
+        c->pToken[i] = (ch>='A' && ch<='Z') ? (ch - 'A' + 'a') : ch;
       }
       *ppToken = c->pToken;
       *pnBytes = n;
diff --git a/third_party/sqlite/src/ext/fts2/mkfts2amal.tcl b/third_party/sqlite/src/ext/fts2/mkfts2amal.tcl
index 5c8d1e9..5c8d1e9 100755..100644
--- a/third_party/sqlite/src/ext/fts2/mkfts2amal.tcl
+++ b/third_party/sqlite/src/ext/fts2/mkfts2amal.tcl
diff --git a/third_party/sqlite/src/ext/fts3/README.tokenizers b/third_party/sqlite/src/ext/fts3/README.tokenizers
index e06803a..e06803a 100755..100644
--- a/third_party/sqlite/src/ext/fts3/README.tokenizers
+++ b/third_party/sqlite/src/ext/fts3/README.tokenizers
diff --git a/third_party/sqlite/src/ext/fts3/fts3.c b/third_party/sqlite/src/ext/fts3/fts3.c
index 3e87a40..91f047e 100644
--- a/third_party/sqlite/src/ext/fts3/fts3.c
+++ b/third_party/sqlite/src/ext/fts3/fts3.c
@@ -23,9 +23,6 @@
 **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
 */
 
-/* TODO(shess) Consider exporting this comment to an HTML file or the
-** wiki.
-*/
 /* The full-text index is stored in a series of b+tree (-like)
 ** structures called segments which map terms to doclists.  The
 ** structures are like b+trees in layout, but are constructed from the
@@ -48,30 +45,40 @@
 ** 21 bits - BBA
 ** and so on.
 **
-** This is identical to how sqlite encodes varints (see util.c).
+** This is similar in concept to how sqlite encodes "varints" but
+** the encoding is not the same.  SQLite varints are big-endian
+** are are limited to 9 bytes in length whereas FTS3 varints are
+** little-endian and can be up to 10 bytes in length (in theory).
+**
+** Example encodings:
+**
+**     1:    0x01
+**   127:    0x7f
+**   128:    0x81 0x00
 **
 **
 **** Document lists ****
 ** A doclist (document list) holds a docid-sorted list of hits for a
-** given term.  Doclists hold docids, and can optionally associate
-** token positions and offsets with docids.
+** given term.  Doclists hold docids and associated token positions.
+** A docid is the unique integer identifier for a single document.
+** A position is the index of a word within the document.  The first 
+** word of the document has a position of 0.
+**
+** FTS3 used to optionally store character offsets using a compile-time
+** option.  But that functionality is no longer supported.
 **
-** A DL_POSITIONS_OFFSETS doclist is stored like this:
+** A doclist is stored like this:
 **
 ** array {
 **   varint docid;
 **   array {                (position list for column 0)
-**     varint position;     (delta from previous position plus POS_BASE)
-**     varint startOffset;  (delta from previous startOffset)
-**     varint endOffset;    (delta from startOffset)
+**     varint position;     (2 more than the delta from previous position)
 **   }
 **   array {
 **     varint POS_COLUMN;   (marks start of position list for new column)
 **     varint column;       (index of new column)
 **     array {
-**       varint position;   (delta from previous position plus POS_BASE)
-**       varint startOffset;(delta from previous startOffset)
-**       varint endOffset;  (delta from startOffset)
+**       varint position;   (2 more than the delta from previous position)
 **     }
 **   }
 **   varint POS_END;        (marks end of positions for this document.
@@ -79,19 +86,32 @@
 **
 ** Here, array { X } means zero or more occurrences of X, adjacent in
 ** memory.  A "position" is an index of a token in the token stream
-** generated by the tokenizer, while an "offset" is a byte offset,
-** both based at 0.  Note that POS_END and POS_COLUMN occur in the
-** same logical place as the position element, and act as sentinals
-** ending a position list array.
-**
-** A DL_POSITIONS doclist omits the startOffset and endOffset
-** information.  A DL_DOCIDS doclist omits both the position and
-** offset information, becoming an array of varint-encoded docids.
-**
-** On-disk data is stored as type DL_DEFAULT, so we don't serialize
-** the type.  Due to how deletion is implemented in the segmentation
-** system, on-disk doclists MUST store at least positions.
-**
+** generated by the tokenizer. Note that POS_END and POS_COLUMN occur 
+** in the same logical place as the position element, and act as sentinals
+** ending a position list array.  POS_END is 0.  POS_COLUMN is 1.
+** The positions numbers are not stored literally but rather as two more
+** than the difference from the prior position, or the just the position plus
+** 2 for the first position.  Example:
+**
+**   label:       A B C D E  F  G H   I  J K
+**   value:     123 5 9 1 1 14 35 0 234 72 0
+**
+** The 123 value is the first docid.  For column zero in this document
+** there are two matches at positions 3 and 10 (5-2 and 9-2+3).  The 1
+** at D signals the start of a new column; the 1 at E indicates that the
+** new column is column number 1.  There are two positions at 12 and 45
+** (14-2 and 35-2+12).  The 0 at H indicate the end-of-document.  The
+** 234 at I is the next docid.  It has one position 72 (72-2) and then
+** terminates with the 0 at K.
+**
+** A "position-list" is the list of positions for multiple columns for
+** a single docid.  A "column-list" is the set of positions for a single
+** column.  Hence, a position-list consists of one or more column-lists,
+** a document record consists of a docid followed by a position-list and
+** a doclist consists of one or more document records.
+**
+** A bare doclist omits the position information, becoming an 
+** array of varint-encoded docids.
 **
 **** Segment leaf nodes ****
 ** Segment leaf nodes store terms and doclists, ordered by term.  Leaf
@@ -271,7 +291,6 @@
 ** deletions and duplications.  This would basically be a forced merge
 ** into a single segment.
 */
-#define CHROMIUM_FTS3_CHANGES 1
 
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
 
@@ -279,41 +298,21 @@
 # define SQLITE_CORE 1
 #endif
 
+#include "fts3Int.h"
+
 #include <assert.h>
 #include <stdlib.h>
+#include <stddef.h>
 #include <stdio.h>
 #include <string.h>
-#include <ctype.h>
+#include <stdarg.h>
 
 #include "fts3.h"
-#include "fts3_expr.h"
-#include "fts3_hash.h"
-#include "fts3_tokenizer.h"
 #ifndef SQLITE_CORE 
 # include "sqlite3ext.h"
   SQLITE_EXTENSION_INIT1
 #endif
 
-
-/* TODO(shess) MAN, this thing needs some refactoring.  At minimum, it
-** would be nice to order the file better, perhaps something along the
-** lines of:
-**
-**  - utility functions
-**  - table setup functions
-**  - table update functions
-**  - table query functions
-**
-** Put the query functions last because they're likely to reference
-** typedefs or functions from the table update section.
-*/
-
-#if 0
-# define FTSTRACE(A)  printf A; fflush(stdout)
-#else
-# define FTSTRACE(A)
-#endif
-
 #if 0
 /* Useful to set breakpoints. See main.c sqlite3Corrupt(). */
 static int fts3Corrupt(void){
@@ -324,80 +323,12 @@ static int fts3Corrupt(void){
 # define SQLITE_CORRUPT_BKPT SQLITE_CORRUPT
 #endif
 
-/* It is not safe to call isspace(), tolower(), or isalnum() on
-** hi-bit-set characters.  This is the same solution used in the
-** tokenizer.
-*/
-/* TODO(shess) The snippet-generation code should be using the
-** tokenizer-generated tokens rather than doing its own local
-** tokenization.
-*/
-/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
-static int safe_isspace(char c){
-  return (c&0x80)==0 ? isspace(c) : 0;
-}
-static int safe_tolower(char c){
-  return (c>='A' && c<='Z') ? (c-'A'+'a') : c;
-}
-static int safe_isalnum(char c){
-  return (c&0x80)==0 ? isalnum(c) : 0;
-}
-
-typedef enum DocListType {
-  DL_DOCIDS,              /* docids only */
-  DL_POSITIONS,           /* docids + positions */
-  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
-} DocListType;
-
-/*
-** By default, only positions and not offsets are stored in the doclists.
-** To change this so that offsets are stored too, compile with
-**
-**          -DDL_DEFAULT=DL_POSITIONS_OFFSETS
-**
-** If DL_DEFAULT is set to DL_DOCIDS, your table can only be inserted
-** into (no deletes or updates).
-*/
-#ifndef DL_DEFAULT
-# define DL_DEFAULT DL_POSITIONS
-#endif
-
-enum {
-  POS_END = 0,        /* end of this position list */
-  POS_COLUMN,         /* followed by new column number */
-  POS_BASE
-};
-
-/* MERGE_COUNT controls how often we merge segments (see comment at
-** top of file).
-*/
-#define MERGE_COUNT 16
-
-/* utility functions */
-
-/* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single
-** record to prevent errors of the form:
-**
-** my_function(SomeType *b){
-**   memset(b, '\0', sizeof(b));  // sizeof(b)!=sizeof(*b)
-** }
+/* 
+** Write a 64-bit variable-length integer to memory starting at p[0].
+** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
+** The number of bytes written is returned.
 */
-/* TODO(shess) Obvious candidates for a header file. */
-#define CLEAR(b) memset(b, '\0', sizeof(*(b)))
-
-#ifndef NDEBUG
-#  define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b)))
-#else
-#  define SCRAMBLE(b)
-#endif
-
-/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
-#define VARINT_MAX 10
-
-/* Write a 64-bit variable-length integer to memory starting at p[0].
- * The length of data written will be between 1 and VARINT_MAX bytes.
- * The number of bytes written is returned. */
-static int fts3PutVarint(char *p, sqlite_int64 v){
+int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
   unsigned char *q = (unsigned char *) p;
   sqlite_uint64 vu = v;
   do{
@@ -405,6908 +336,2328 @@ static int fts3PutVarint(char *p, sqlite_int64 v){
     vu >>= 7;
   }while( vu!=0 );
   q[-1] &= 0x7f;  /* turn off high bit in final byte */
-  assert( q - (unsigned char *)p <= VARINT_MAX );
+  assert( q - (unsigned char *)p <= FTS3_VARINT_MAX );
   return (int) (q - (unsigned char *)p);
 }
 
-/* Read a 64-bit variable-length integer from memory starting at p[0].
- * Return the number of bytes read, or 0 on error.
- * The value is stored in *v. */
-static int fts3GetVarintSafe(const char *p, sqlite_int64 *v, int max){
+/* 
+** Read a 64-bit variable-length integer from memory starting at p[0].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
   const unsigned char *q = (const unsigned char *) p;
   sqlite_uint64 x = 0, y = 1;
-  if( max>VARINT_MAX ) max = VARINT_MAX;
-  while( max && (*q & 0x80) == 0x80 ){
-    max--;
+  while( (*q&0x80)==0x80 && q-(unsigned char *)p<FTS3_VARINT_MAX ){
     x += y * (*q++ & 0x7f);
     y <<= 7;
   }
-  if( !max ){
-    assert( 0 );
-    return 0;  /* tried to read too much; bad data */
-  }
   x += y * (*q++);
   *v = (sqlite_int64) x;
   return (int) (q - (unsigned char *)p);
 }
 
-static int fts3GetVarint(const char *p, sqlite_int64 *v){
-  return fts3GetVarintSafe(p, v, VARINT_MAX);
-}
-
-static int fts3GetVarint32Safe(const char *p, int *pi, int max){
+/*
+** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
+** 32-bit integer before it is returned.
+*/
+int sqlite3Fts3GetVarint32(const char *p, int *pi){
  sqlite_int64 i;
- int ret = fts3GetVarintSafe(p, &i, max);
- if( !ret ) return ret;
+ int ret = sqlite3Fts3GetVarint(p, &i);
  *pi = (int) i;
- assert( *pi==i );
  return ret;
 }
 
-static int fts3GetVarint32(const char* p, int *pi){
-  return fts3GetVarint32Safe(p, pi, VARINT_MAX);
-}
-
-/*******************************************************************/
-/* DataBuffer is used to collect data into a buffer in piecemeal
-** fashion.  It implements the usual distinction between amount of
-** data currently stored (nData) and buffer capacity (nCapacity).
-**
-** dataBufferInit - create a buffer with given initial capacity.
-** dataBufferReset - forget buffer's data, retaining capacity.
-** dataBufferDestroy - free buffer's data.
-** dataBufferSwap - swap contents of two buffers.
-** dataBufferExpand - expand capacity without adding data.
-** dataBufferAppend - append data.
-** dataBufferAppend2 - append two pieces of data at once.
-** dataBufferReplace - replace buffer's data.
-*/
-typedef struct DataBuffer {
-  char *pData;          /* Pointer to malloc'ed buffer. */
-  int nCapacity;        /* Size of pData buffer. */
-  int nData;            /* End of data loaded into pData. */
-} DataBuffer;
-
-static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
-  assert( nCapacity>=0 );
-  pBuffer->nData = 0;
-  pBuffer->nCapacity = nCapacity;
-  pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
-}
-static void dataBufferReset(DataBuffer *pBuffer){
-  pBuffer->nData = 0;
-}
-static void dataBufferDestroy(DataBuffer *pBuffer){
-  if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData);
-  SCRAMBLE(pBuffer);
-}
-static void dataBufferSwap(DataBuffer *pBuffer1, DataBuffer *pBuffer2){
-  DataBuffer tmp = *pBuffer1;
-  *pBuffer1 = *pBuffer2;
-  *pBuffer2 = tmp;
-}
-static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
-  assert( nAddCapacity>0 );
-  /* TODO(shess) Consider expanding more aggressively.  Note that the
-  ** underlying malloc implementation may take care of such things for
-  ** us already.
-  */
-  if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
-    pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
-    pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
-  }
-}
-static void dataBufferAppend(DataBuffer *pBuffer,
-                             const char *pSource, int nSource){
-  assert( nSource>0 && pSource!=NULL );
-  dataBufferExpand(pBuffer, nSource);
-  memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
-  pBuffer->nData += nSource;
-}
-static void dataBufferAppend2(DataBuffer *pBuffer,
-                              const char *pSource1, int nSource1,
-                              const char *pSource2, int nSource2){
-  assert( nSource1>0 && pSource1!=NULL );
-  assert( nSource2>0 && pSource2!=NULL );
-  dataBufferExpand(pBuffer, nSource1+nSource2);
-  memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
-  memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
-  pBuffer->nData += nSource1+nSource2;
-}
-static void dataBufferReplace(DataBuffer *pBuffer,
-                              const char *pSource, int nSource){
-  dataBufferReset(pBuffer);
-  dataBufferAppend(pBuffer, pSource, nSource);
-}
-
-/* StringBuffer is a null-terminated version of DataBuffer. */
-typedef struct StringBuffer {
-  DataBuffer b;            /* Includes null terminator. */
-} StringBuffer;
-
-static void initStringBuffer(StringBuffer *sb){
-  dataBufferInit(&sb->b, 100);
-  dataBufferReplace(&sb->b, "", 1);
-}
-static int stringBufferLength(StringBuffer *sb){
-  return sb->b.nData-1;
-}
-static char *stringBufferData(StringBuffer *sb){
-  return sb->b.pData;
-}
-static void stringBufferDestroy(StringBuffer *sb){
-  dataBufferDestroy(&sb->b);
-}
-
-static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){
-  assert( sb->b.nData>0 );
-  if( nFrom>0 ){
-    sb->b.nData--;
-    dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
-  }
-}
-static void append(StringBuffer *sb, const char *zFrom){
-  nappend(sb, zFrom, strlen(zFrom));
-}
-
-/* Append a list of strings separated by commas. */
-static void appendList(StringBuffer *sb, int nString, char **azString){
-  int i;
-  for(i=0; i<nString; ++i){
-    if( i>0 ) append(sb, ", ");
-    append(sb, azString[i]);
-  }
-}
-
-static int endsInWhiteSpace(StringBuffer *p){
-  return stringBufferLength(p)>0 &&
-    safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
-}
-
-/* If the StringBuffer ends in something other than white space, add a
-** single space character to the end.
+/*
+** Return the number of bytes required to encode v as a varint
 */
-static void appendWhiteSpace(StringBuffer *p){
-  if( stringBufferLength(p)==0 ) return;
-  if( !endsInWhiteSpace(p) ) append(p, " ");
-}
-
-/* Remove white space from the end of the StringBuffer */
-static void trimWhiteSpace(StringBuffer *p){
-  while( endsInWhiteSpace(p) ){
-    p->b.pData[--p->b.nData-1] = '\0';
-  }
+int sqlite3Fts3VarintLen(sqlite3_uint64 v){
+  int i = 0;
+  do{
+    i++;
+    v >>= 7;
+  }while( v!=0 );
+  return i;
 }
 
-/*******************************************************************/
-/* DLReader is used to read document elements from a doclist.  The
-** current docid is cached, so dlrDocid() is fast.  DLReader does not
-** own the doclist buffer.
+/*
+** Convert an SQL-style quoted string into a normal string by removing
+** the quote characters.  The conversion is done in-place.  If the
+** input does not begin with a quote character, then this routine
+** is a no-op.
 **
-** dlrAtEnd - true if there's no more data to read.
-** dlrDocid - docid of current document.
-** dlrDocData - doclist data for current document (including docid).
-** dlrDocDataBytes - length of same.
-** dlrAllDataBytes - length of all remaining data.
-** dlrPosData - position data for current document.
-** dlrPosDataLen - length of pos data for current document (incl POS_END).
-** dlrStep - step to current document.
-** dlrInit - initial for doclist of given type against given data.
-** dlrDestroy - clean up.
+** Examples:
 **
-** Expected usage is something like:
+**     "abc"   becomes   abc
+**     'xyz'   becomes   xyz
+**     [pqr]   becomes   pqr
+**     `mno`   becomes   mno
 **
-**   DLReader reader;
-**   dlrInit(&reader, pData, nData);
-**   while( !dlrAtEnd(&reader) ){
-**     // calls to dlrDocid() and kin.
-**     dlrStep(&reader);
-**   }
-**   dlrDestroy(&reader);
-*/
-typedef struct DLReader {
-  DocListType iType;
-  const char *pData;
-  int nData;
-
-  sqlite_int64 iDocid;
-  int nElement;
-} DLReader;
-
-static int dlrAtEnd(DLReader *pReader){
-  assert( pReader->nData>=0 );
-  return pReader->nData<=0;
-}
-static sqlite_int64 dlrDocid(DLReader *pReader){
-  assert( !dlrAtEnd(pReader) );
-  return pReader->iDocid;
-}
-static const char *dlrDocData(DLReader *pReader){
-  assert( !dlrAtEnd(pReader) );
-  return pReader->pData;
-}
-static int dlrDocDataBytes(DLReader *pReader){
-  assert( !dlrAtEnd(pReader) );
-  return pReader->nElement;
-}
-static int dlrAllDataBytes(DLReader *pReader){
-  assert( !dlrAtEnd(pReader) );
-  return pReader->nData;
-}
-/* TODO(shess) Consider adding a field to track iDocid varint length
-** to make these two functions faster.  This might matter (a tiny bit)
-** for queries.
 */
-static const char *dlrPosData(DLReader *pReader){
-  sqlite_int64 iDummy;
-  int n = fts3GetVarintSafe(pReader->pData, &iDummy, pReader->nElement);
-  if( !n ) return NULL;
-  assert( !dlrAtEnd(pReader) );
-  return pReader->pData+n;
-}
-static int dlrPosDataLen(DLReader *pReader){
-  sqlite_int64 iDummy;
-  int n = fts3GetVarint(pReader->pData, &iDummy);
-  assert( !dlrAtEnd(pReader) );
-  return pReader->nElement-n;
-}
-static int dlrStep(DLReader *pReader){
-  assert( !dlrAtEnd(pReader) );
-
-  /* Skip past current doclist element. */
-  assert( pReader->nElement<=pReader->nData );
-  pReader->pData += pReader->nElement;
-  pReader->nData -= pReader->nElement;
-
-  /* If there is more data, read the next doclist element. */
-  if( pReader->nData>0 ){
-    sqlite_int64 iDocidDelta;
-    int nTotal = 0;
-    int iDummy, n = fts3GetVarintSafe(pReader->pData, &iDocidDelta, pReader->nData);
-    if( !n ) return SQLITE_CORRUPT_BKPT;
-    nTotal += n;
-    pReader->iDocid += iDocidDelta;
-    if( pReader->iType>=DL_POSITIONS ){
-      while( 1 ){
-        n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal);
-        if( !n ) return SQLITE_CORRUPT_BKPT;
-        nTotal += n;
-        if( iDummy==POS_END ) break;
-        if( iDummy==POS_COLUMN ){
-          n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal);
-          if( !n ) return SQLITE_CORRUPT_BKPT;
-          nTotal += n;
-        }else if( pReader->iType==DL_POSITIONS_OFFSETS ){
-          n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal);
-          if( !n ) return SQLITE_CORRUPT_BKPT;
-          nTotal += n;
-          n = fts3GetVarint32Safe(pReader->pData+nTotal, &iDummy, pReader->nData-nTotal);
-          if( !n ) return SQLITE_CORRUPT_BKPT;
-          nTotal += n;
-        }
-      }
-    }
-    pReader->nElement = nTotal;
-    assert( pReader->nElement<=pReader->nData );
-  }
-  return SQLITE_OK;
-}
-static void dlrDestroy(DLReader *pReader){
-  SCRAMBLE(pReader);
-}
-static int dlrInit(DLReader *pReader, DocListType iType,
-                   const char *pData, int nData){
-  int rc;
-  assert( pData!=NULL && nData!=0 );
-  pReader->iType = iType;
-  pReader->pData = pData;
-  pReader->nData = nData;
-  pReader->nElement = 0;
-  pReader->iDocid = 0;
-
-  /* Load the first element's data.  There must be a first element. */
-  rc = dlrStep(pReader);
-  if( rc!=SQLITE_OK ) dlrDestroy(pReader);
-  return rc;
-}
+void sqlite3Fts3Dequote(char *z){
+  char quote;                     /* Quote character (if any ) */
 
-#ifndef NDEBUG
-/* Verify that the doclist can be validly decoded.  Also returns the
-** last docid found because it is convenient in other assertions for
-** DLWriter.
-*/
-static void docListValidate(DocListType iType, const char *pData, int nData,
-                            sqlite_int64 *pLastDocid){
-  sqlite_int64 iPrevDocid = 0;
-  assert( nData>0 );
-  assert( pData!=0 );
-  assert( pData+nData>pData );
-  while( nData!=0 ){
-    sqlite_int64 iDocidDelta;
-    int n = fts3GetVarint(pData, &iDocidDelta);
-    iPrevDocid += iDocidDelta;
-    if( iType>DL_DOCIDS ){
-      int iDummy;
-      while( 1 ){
-        n += fts3GetVarint32(pData+n, &iDummy);
-        if( iDummy==POS_END ) break;
-        if( iDummy==POS_COLUMN ){
-          n += fts3GetVarint32(pData+n, &iDummy);
-        }else if( iType>DL_POSITIONS ){
-          n += fts3GetVarint32(pData+n, &iDummy);
-          n += fts3GetVarint32(pData+n, &iDummy);
-        }
-        assert( n<=nData );
+  quote = z[0];
+  if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
+    int iIn = 1;                  /* Index of next byte to read from input */
+    int iOut = 0;                 /* Index of next byte to write to output */
+
+    /* If the first byte was a '[', then the close-quote character is a ']' */
+    if( quote=='[' ) quote = ']';  
+
+    while( ALWAYS(z[iIn]) ){
+      if( z[iIn]==quote ){
+        if( z[iIn+1]!=quote ) break;
+        z[iOut++] = quote;
+        iIn += 2;
+      }else{
+        z[iOut++] = z[iIn++];
       }
     }
-    assert( n<=nData );
-    pData += n;
-    nData -= n;
+    z[iOut] = '\0';
   }
-  if( pLastDocid ) *pLastDocid = iPrevDocid;
 }
-#define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o)
-#else
-#define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 )
-#endif
 
-/*******************************************************************/
-/* DLWriter is used to write doclist data to a DataBuffer.  DLWriter
-** always appends to the buffer and does not own it.
-**
-** dlwInit - initialize to write a given type doclistto a buffer.
-** dlwDestroy - clear the writer's memory.  Does not free buffer.
-** dlwAppend - append raw doclist data to buffer.
-** dlwCopy - copy next doclist from reader to writer.
-** dlwAdd - construct doclist element and append to buffer.
-**    Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter).
+/*
+** Read a single varint from the doclist at *pp and advance *pp to point
+** to the first byte past the end of the varint.  Add the value of the varint
+** to *pVal.
 */
-typedef struct DLWriter {
-  DocListType iType;
-  DataBuffer *b;
-  sqlite_int64 iPrevDocid;
-#ifndef NDEBUG
-  int has_iPrevDocid;
-#endif
-} DLWriter;
-
-static void dlwInit(DLWriter *pWriter, DocListType iType, DataBuffer *b){
-  pWriter->b = b;
-  pWriter->iType = iType;
-  pWriter->iPrevDocid = 0;
-#ifndef NDEBUG
-  pWriter->has_iPrevDocid = 0;
-#endif
-}
-static void dlwDestroy(DLWriter *pWriter){
-  SCRAMBLE(pWriter);
+static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){
+  sqlite3_int64 iVal;
+  *pp += sqlite3Fts3GetVarint(*pp, &iVal);
+  *pVal += iVal;
 }
-/* iFirstDocid is the first docid in the doclist in pData.  It is
-** needed because pData may point within a larger doclist, in which
-** case the first item would be delta-encoded.
-**
-** iLastDocid is the final docid in the doclist in pData.  It is
-** needed to create the new iPrevDocid for future delta-encoding.  The
-** code could decode the passed doclist to recreate iLastDocid, but
-** the only current user (docListMerge) already has decoded this
-** information.
-*/
-/* TODO(shess) This has become just a helper for docListMerge.
-** Consider a refactor to make this cleaner.
-*/
-static int dlwAppend(DLWriter *pWriter,
-                     const char *pData, int nData,
-                     sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
-  sqlite_int64 iDocid = 0;
-  char c[VARINT_MAX];
-  int nFirstOld, nFirstNew;     /* Old and new varint len of first docid. */
-#ifndef NDEBUG
-  sqlite_int64 iLastDocidDelta;
-#endif
-
-  /* Recode the initial docid as delta from iPrevDocid. */
-  nFirstOld = fts3GetVarintSafe(pData, &iDocid, nData);
-  if( !nFirstOld ) return SQLITE_CORRUPT_BKPT;
-  assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
-  nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid);
-
-  /* Verify that the incoming doclist is valid AND that it ends with
-  ** the expected docid.  This is essential because we'll trust this
-  ** docid in future delta-encoding.
-  */
-  ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta);
-  assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta );
 
-  /* Append recoded initial docid and everything else.  Rest of docids
-  ** should have been delta-encoded from previous initial docid.
-  */
-  if( nFirstOld<nData ){
-    dataBufferAppend2(pWriter->b, c, nFirstNew,
-                      pData+nFirstOld, nData-nFirstOld);
+/*
+** As long as *pp has not reached its end (pEnd), then do the same
+** as fts3GetDeltaVarint(): read a single varint and add it to *pVal.
+** But if we have reached the end of the varint, just set *pp=0 and
+** leave *pVal unchanged.
+*/
+static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){
+  if( *pp>=pEnd ){
+    *pp = 0;
   }else{
-    dataBufferAppend(pWriter->b, c, nFirstNew);
+    fts3GetDeltaVarint(pp, pVal);
   }
-  pWriter->iPrevDocid = iLastDocid;
-  return SQLITE_OK;
-}
-static int dlwCopy(DLWriter *pWriter, DLReader *pReader){
-  return dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
-                   dlrDocid(pReader), dlrDocid(pReader));
-}
-static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
-  char c[VARINT_MAX];
-  int n = fts3PutVarint(c, iDocid-pWriter->iPrevDocid);
-
-  /* Docids must ascend. */
-  assert( !pWriter->has_iPrevDocid || iDocid>pWriter->iPrevDocid );
-  assert( pWriter->iType==DL_DOCIDS );
-
-  dataBufferAppend(pWriter->b, c, n);
-  pWriter->iPrevDocid = iDocid;
-#ifndef NDEBUG
-  pWriter->has_iPrevDocid = 1;
-#endif
 }
 
-/*******************************************************************/
-/* PLReader is used to read data from a document's position list.  As
-** the caller steps through the list, data is cached so that varints
-** only need to be decoded once.
-**
-** plrInit, plrDestroy - create/destroy a reader.
-** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors
-** plrAtEnd - at end of stream, only call plrDestroy once true.
-** plrStep - step to the next element.
+/*
+** The xDisconnect() virtual table method.
 */
-typedef struct PLReader {
-  /* These refer to the next position's data.  nData will reach 0 when
-  ** reading the last position, so plrStep() signals EOF by setting
-  ** pData to NULL.
-  */
-  const char *pData;
-  int nData;
-
-  DocListType iType;
-  int iColumn;         /* the last column read */
-  int iPosition;       /* the last position read */
-  int iStartOffset;    /* the last start offset read */
-  int iEndOffset;      /* the last end offset read */
-} PLReader;
-
-static int plrAtEnd(PLReader *pReader){
-  return pReader->pData==NULL;
-}
-static int plrColumn(PLReader *pReader){
-  assert( !plrAtEnd(pReader) );
-  return pReader->iColumn;
-}
-static int plrPosition(PLReader *pReader){
-  assert( !plrAtEnd(pReader) );
-  return pReader->iPosition;
-}
-static int plrStartOffset(PLReader *pReader){
-  assert( !plrAtEnd(pReader) );
-  return pReader->iStartOffset;
-}
-static int plrEndOffset(PLReader *pReader){
-  assert( !plrAtEnd(pReader) );
-  return pReader->iEndOffset;
-}
-static int plrStep(PLReader *pReader){
-  int i, n, nTotal = 0;
+static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
+  Fts3Table *p = (Fts3Table *)pVtab;
+  int i;
 
-  assert( !plrAtEnd(pReader) );
+  assert( p->nPendingData==0 );
 
-  if( pReader->nData<=0 ){
-    pReader->pData = NULL;
-    return SQLITE_OK;
+  /* Free any prepared statements held */
+  for(i=0; i<SizeofArray(p->aStmt); i++){
+    sqlite3_finalize(p->aStmt[i]);
   }
-
-  n = fts3GetVarint32Safe(pReader->pData, &i, pReader->nData);
-  if( !n ) return SQLITE_CORRUPT_BKPT;
-  nTotal += n;
-  if( i==POS_COLUMN ){
-    n = fts3GetVarint32Safe(pReader->pData+nTotal, &pReader->iColumn, pReader->nData-nTotal);
-    if( !n ) return SQLITE_CORRUPT_BKPT;
-    nTotal += n;
-    pReader->iPosition = 0;
-    pReader->iStartOffset = 0;
-    n = fts3GetVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal);
-    if( !n ) return SQLITE_CORRUPT_BKPT;
-    nTotal += n;
+  for(i=0; i<p->nLeavesStmt; i++){
+    sqlite3_finalize(p->aLeavesStmt[i]);
   }
-  /* Should never see adjacent column changes. */
-  assert( i!=POS_COLUMN );
+  sqlite3_free(p->zSelectLeaves);
+  sqlite3_free(p->aLeavesStmt);
 
-  if( i==POS_END ){
-    assert( nTotal<=pReader->nData );
-    pReader->nData = 0;
-    pReader->pData = NULL;
-    return SQLITE_OK;
-  }
+  /* Invoke the tokenizer destructor to free the tokenizer. */
+  p->pTokenizer->pModule->xDestroy(p->pTokenizer);
 
-  pReader->iPosition += i-POS_BASE;
-  if( pReader->iType==DL_POSITIONS_OFFSETS ){
-    n = fts3GetVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal);
-    if( !n ) return SQLITE_CORRUPT_BKPT;
-    nTotal += n;
-    pReader->iStartOffset += i;
-    n = fts3GetVarint32Safe(pReader->pData+nTotal, &i, pReader->nData-nTotal);
-    if( !n ) return SQLITE_CORRUPT_BKPT;
-    nTotal += n;
-    pReader->iEndOffset = pReader->iStartOffset+i;
-  }
-  assert( nTotal<=pReader->nData );
-  pReader->pData += nTotal;
-  pReader->nData -= nTotal;
+  sqlite3_free(p);
   return SQLITE_OK;
 }
 
-static void plrDestroy(PLReader *pReader){
-  SCRAMBLE(pReader);
-}
-static int plrInit(PLReader *pReader, DLReader *pDLReader){
-  int rc;
-  pReader->pData = dlrPosData(pDLReader);
-  pReader->nData = dlrPosDataLen(pDLReader);
-  pReader->iType = pDLReader->iType;
-  pReader->iColumn = 0;
-  pReader->iPosition = 0;
-  pReader->iStartOffset = 0;
-  pReader->iEndOffset = 0;
-  rc = plrStep(pReader);
-  if( rc!=SQLITE_OK ) plrDestroy(pReader);
-  return rc;
-}
-
-/*******************************************************************/
-/* PLWriter is used in constructing a document's position list.  As a
-** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op.
-** PLWriter writes to the associated DLWriter's buffer.
-**
-** plwInit - init for writing a document's poslist.
-** plwDestroy - clear a writer.
-** plwAdd - append position and offset information.
-** plwCopy - copy next position's data from reader to writer.
-** plwTerminate - add any necessary doclist terminator.
-**
-** Calling plwAdd() after plwTerminate() may result in a corrupt
-** doclist.
-*/
-/* TODO(shess) Until we've written the second item, we can cache the
-** first item's information.  Then we'd have three states:
-**
-** - initialized with docid, no positions.
-** - docid and one position.
-** - docid and multiple positions.
-**
-** Only the last state needs to actually write to dlw->b, which would
-** be an improvement in the DLCollector case.
-*/
-typedef struct PLWriter {
-  DLWriter *dlw;
-
-  int iColumn;    /* the last column written */
-  int iPos;       /* the last position written */
-  int iOffset;    /* the last start offset written */
-} PLWriter;
-
-/* TODO(shess) In the case where the parent is reading these values
-** from a PLReader, we could optimize to a copy if that PLReader has
-** the same type as pWriter.
-*/
-static void plwAdd(PLWriter *pWriter, int iColumn, int iPos,
-                   int iStartOffset, int iEndOffset){
-  /* Worst-case space for POS_COLUMN, iColumn, iPosDelta,
-  ** iStartOffsetDelta, and iEndOffsetDelta.
-  */
-  char c[5*VARINT_MAX];
-  int n = 0;
-
-  /* Ban plwAdd() after plwTerminate(). */
-  assert( pWriter->iPos!=-1 );
-
-  if( pWriter->dlw->iType==DL_DOCIDS ) return;
-
-  if( iColumn!=pWriter->iColumn ){
-    n += fts3PutVarint(c+n, POS_COLUMN);
-    n += fts3PutVarint(c+n, iColumn);
-    pWriter->iColumn = iColumn;
-    pWriter->iPos = 0;
-    pWriter->iOffset = 0;
-  }
-  assert( iPos>=pWriter->iPos );
-  n += fts3PutVarint(c+n, POS_BASE+(iPos-pWriter->iPos));
-  pWriter->iPos = iPos;
-  if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){
-    assert( iStartOffset>=pWriter->iOffset );
-    n += fts3PutVarint(c+n, iStartOffset-pWriter->iOffset);
-    pWriter->iOffset = iStartOffset;
-    assert( iEndOffset>=iStartOffset );
-    n += fts3PutVarint(c+n, iEndOffset-iStartOffset);
-  }
-  dataBufferAppend(pWriter->dlw->b, c, n);
-}
-static void plwCopy(PLWriter *pWriter, PLReader *pReader){
-  plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader),
-         plrStartOffset(pReader), plrEndOffset(pReader));
-}
-static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid){
-  char c[VARINT_MAX];
-  int n;
-
-  pWriter->dlw = dlw;
-
-  /* Docids must ascend. */
-  assert( !pWriter->dlw->has_iPrevDocid || iDocid>pWriter->dlw->iPrevDocid );
-  n = fts3PutVarint(c, iDocid-pWriter->dlw->iPrevDocid);
-  dataBufferAppend(pWriter->dlw->b, c, n);
-  pWriter->dlw->iPrevDocid = iDocid;
-#ifndef NDEBUG
-  pWriter->dlw->has_iPrevDocid = 1;
-#endif
-
-  pWriter->iColumn = 0;
-  pWriter->iPos = 0;
-  pWriter->iOffset = 0;
-}
-/* TODO(shess) Should plwDestroy() also terminate the doclist?  But
-** then plwDestroy() would no longer be just a destructor, it would
-** also be doing work, which isn't consistent with the overall idiom.
-** Another option would be for plwAdd() to always append any necessary
-** terminator, so that the output is always correct.  But that would
-** add incremental work to the common case with the only benefit being
-** API elegance.  Punt for now.
-*/
-static void plwTerminate(PLWriter *pWriter){
-  if( pWriter->dlw->iType>DL_DOCIDS ){
-    char c[VARINT_MAX];
-    int n = fts3PutVarint(c, POS_END);
-    dataBufferAppend(pWriter->dlw->b, c, n);
-  }
-#ifndef NDEBUG
-  /* Mark as terminated for assert in plwAdd(). */
-  pWriter->iPos = -1;
-#endif
-}
-static void plwDestroy(PLWriter *pWriter){
-  SCRAMBLE(pWriter);
-}
-
-/*******************************************************************/
-/* DLCollector wraps PLWriter and DLWriter to provide a
-** dynamically-allocated doclist area to use during tokenization.
+/*
+** Construct one or more SQL statements from the format string given
+** and then evaluate those statements.  The success code is writting
+** into *pRc.
 **
-** dlcNew - malloc up and initialize a collector.
-** dlcDelete - destroy a collector and all contained items.
-** dlcAddPos - append position and offset information.
-** dlcAddDoclist - add the collected doclist to the given buffer.
-** dlcNext - terminate the current document and open another.
-*/
-typedef struct DLCollector {
-  DataBuffer b;
-  DLWriter dlw;
-  PLWriter plw;
-} DLCollector;
-
-/* TODO(shess) This could also be done by calling plwTerminate() and
-** dataBufferAppend().  I tried that, expecting nominal performance
-** differences, but it seemed to pretty reliably be worth 1% to code
-** it this way.  I suspect it is the incremental malloc overhead (some
-** percentage of the plwTerminate() calls will cause a realloc), so
-** this might be worth revisiting if the DataBuffer implementation
-** changes.
+** If *pRc is initially non-zero then this routine is a no-op.
 */
-static void dlcAddDoclist(DLCollector *pCollector, DataBuffer *b){
-  if( pCollector->dlw.iType>DL_DOCIDS ){
-    char c[VARINT_MAX];
-    int n = fts3PutVarint(c, POS_END);
-    dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n);
+static void fts3DbExec(
+  int *pRc,              /* Success code */
+  sqlite3 *db,           /* Database in which to run SQL */
+  const char *zFormat,   /* Format string for SQL */
+  ...                    /* Arguments to the format string */
+){
+  va_list ap;
+  char *zSql;
+  if( *pRc ) return;
+  va_start(ap, zFormat);
+  zSql = sqlite3_vmprintf(zFormat, ap);
+  va_end(ap);
+  if( zSql==0 ){
+    *pRc = SQLITE_NOMEM;
   }else{
-    dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData);
-  }
-}
-static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){
-  plwTerminate(&pCollector->plw);
-  plwDestroy(&pCollector->plw);
-  plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
-}
-static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos,
-                      int iStartOffset, int iEndOffset){
-  plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset);
-}
-
-static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){
-  DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector));
-  dataBufferInit(&pCollector->b, 0);
-  dlwInit(&pCollector->dlw, iType, &pCollector->b);
-  plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
-  return pCollector;
-}
-static void dlcDelete(DLCollector *pCollector){
-  plwDestroy(&pCollector->plw);
-  dlwDestroy(&pCollector->dlw);
-  dataBufferDestroy(&pCollector->b);
-  SCRAMBLE(pCollector);
-  sqlite3_free(pCollector);
-}
-
-
-/* Copy the doclist data of iType in pData/nData into *out, trimming
-** unnecessary data as we go.  Only columns matching iColumn are
-** copied, all columns copied if iColumn is -1.  Elements with no
-** matching columns are dropped.  The output is an iOutType doclist.
-*/
-/* NOTE(shess) This code is only valid after all doclists are merged.
-** If this is run before merges, then doclist items which represent
-** deletion will be trimmed, and will thus not effect a deletion
-** during the merge.
-*/
-static int docListTrim(DocListType iType, const char *pData, int nData,
-                       int iColumn, DocListType iOutType, DataBuffer *out){
-  DLReader dlReader;
-  DLWriter dlWriter;
-  int rc;
-
-  assert( iOutType<=iType );
-
-  rc = dlrInit(&dlReader, iType, pData, nData);
-  if( rc!=SQLITE_OK ) return rc;
-  dlwInit(&dlWriter, iOutType, out);
-
-  while( !dlrAtEnd(&dlReader) ){
-    PLReader plReader;
-    PLWriter plWriter;
-    int match = 0;
-
-    rc = plrInit(&plReader, &dlReader);
-    if( rc!=SQLITE_OK ) break;
-
-    while( !plrAtEnd(&plReader) ){
-      if( iColumn==-1 || plrColumn(&plReader)==iColumn ){
-        if( !match ){
-          plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader));
-          match = 1;
-        }
-        plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
-               plrStartOffset(&plReader), plrEndOffset(&plReader));
-      }
-      rc = plrStep(&plReader);
-      if( rc!=SQLITE_OK ){
-        plrDestroy(&plReader);
-        goto err;
-      }
-    }
-    if( match ){
-      plwTerminate(&plWriter);
-      plwDestroy(&plWriter);
-    }
-
-    plrDestroy(&plReader);
-    rc = dlrStep(&dlReader);
-    if( rc!=SQLITE_OK ) break;
+    *pRc = sqlite3_exec(db, zSql, 0, 0, 0);
+    sqlite3_free(zSql);
   }
-err:
-  dlwDestroy(&dlWriter);
-  dlrDestroy(&dlReader);
-  return rc;
 }
 
-/* Used by docListMerge() to keep doclists in the ascending order by
-** docid, then ascending order by age (so the newest comes first).
-*/
-typedef struct OrderedDLReader {
-  DLReader *pReader;
-
-  /* TODO(shess) If we assume that docListMerge pReaders is ordered by
-  ** age (which we do), then we could use pReader comparisons to break
-  ** ties.
+/*
+** The xDestroy() virtual table method.
+*/
+static int fts3DestroyMethod(sqlite3_vtab *pVtab){
+  int rc = SQLITE_OK;              /* Return code */
+  Fts3Table *p = (Fts3Table *)pVtab;
+  sqlite3 *db = p->db;
+
+  /* Drop the shadow tables */
+  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName);
+  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName);
+  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName);
+  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName);
+  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName);
+
+  /* If everything has worked, invoke fts3DisconnectMethod() to free the
+  ** memory associated with the Fts3Table structure and return SQLITE_OK.
+  ** Otherwise, return an SQLite error code.
   */
-  int idx;
-} OrderedDLReader;
-
-/* Order eof to end, then by docid asc, idx desc. */
-static int orderedDLReaderCmp(OrderedDLReader *r1, OrderedDLReader *r2){
-  if( dlrAtEnd(r1->pReader) ){
-    if( dlrAtEnd(r2->pReader) ) return 0;  /* Both atEnd(). */
-    return 1;                              /* Only r1 atEnd(). */
-  }
-  if( dlrAtEnd(r2->pReader) ) return -1;   /* Only r2 atEnd(). */
-
-  if( dlrDocid(r1->pReader)<dlrDocid(r2->pReader) ) return -1;
-  if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1;
-
-  /* Descending on idx. */
-  return r2->idx-r1->idx;
+  return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc);
 }
 
-/* Bubble p[0] to appropriate place in p[1..n-1].  Assumes that
-** p[1..n-1] is already sorted.
-*/
-/* TODO(shess) Is this frequent enough to warrant a binary search?
-** Before implementing that, instrument the code to check.  In most
-** current usage, I expect that p[0] will be less than p[1] a very
-** high proportion of the time.
-*/
-static void orderedDLReaderReorder(OrderedDLReader *p, int n){
-  while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){
-    OrderedDLReader tmp = p[0];
-    p[0] = p[1];
-    p[1] = tmp;
-    n--;
-    p++;
-  }
-}
 
-/* Given an array of doclist readers, merge their doclist elements
-** into out in sorted order (by docid), dropping elements from older
-** readers when there is a duplicate docid.  pReaders is assumed to be
-** ordered by age, oldest first.
-*/
-/* TODO(shess) nReaders must be <= MERGE_COUNT.  This should probably
-** be fixed.
+/*
+** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
+** passed as the first argument. This is done as part of the xConnect()
+** and xCreate() methods.
 */
-static int docListMerge(DataBuffer *out,
-                        DLReader *pReaders, int nReaders){
-  OrderedDLReader readers[MERGE_COUNT];
-  DLWriter writer;
-  int i, n;
-  const char *pStart = 0;
-  int nStart = 0;
-  sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
-  int rc = SQLITE_OK;
+static int fts3DeclareVtab(Fts3Table *p){
+  int i;                          /* Iterator variable */
+  int rc;                         /* Return code */
+  char *zSql;                     /* SQL statement passed to declare_vtab() */
+  char *zCols;                    /* List of user defined columns */
 
-  assert( nReaders>0 );
-  if( nReaders==1 ){
-    dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
-    return SQLITE_OK;
+  /* Create a list of user columns for the virtual table */
+  zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
+  for(i=1; zCols && i<p->nColumn; i++){
+    zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
   }
 
-  assert( nReaders<=MERGE_COUNT );
-  n = 0;
-  for(i=0; i<nReaders; i++){
-    assert( pReaders[i].iType==pReaders[0].iType );
-    readers[i].pReader = pReaders+i;
-    readers[i].idx = i;
-    n += dlrAllDataBytes(&pReaders[i]);
-  }
-  /* Conservatively size output to sum of inputs.  Output should end
-  ** up strictly smaller than input.
-  */
-  dataBufferExpand(out, n);
-
-  /* Get the readers into sorted order. */
-  while( i-->0 ){
-    orderedDLReaderReorder(readers+i, nReaders-i);
-  }
-
-  dlwInit(&writer, pReaders[0].iType, out);
-  while( !dlrAtEnd(readers[0].pReader) ){
-    sqlite_int64 iDocid = dlrDocid(readers[0].pReader);
-
-    /* If this is a continuation of the current buffer to copy, extend
-    ** that buffer.  memcpy() seems to be more efficient if it has a
-    ** lots of data to copy.
-    */
-    if( dlrDocData(readers[0].pReader)==pStart+nStart ){
-      nStart += dlrDocDataBytes(readers[0].pReader);
-    }else{
-      if( pStart!=0 ){
-        rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
-        if( rc!=SQLITE_OK ) goto err;
-      }
-      pStart = dlrDocData(readers[0].pReader);
-      nStart = dlrDocDataBytes(readers[0].pReader);
-      iFirstDocid = iDocid;
-    }
-    iLastDocid = iDocid;
-    rc = dlrStep(readers[0].pReader);
-    if( rc!= SQLITE_OK ) goto err;
-
-    /* Drop all of the older elements with the same docid. */
-    for(i=1; i<nReaders &&
-             !dlrAtEnd(readers[i].pReader) &&
-             dlrDocid(readers[i].pReader)==iDocid; i++){
-      rc = dlrStep(readers[i].pReader);
-      if( rc!=SQLITE_OK ) goto err;
-    }
+  /* Create the whole "CREATE TABLE" statement to pass to SQLite */
+  zSql = sqlite3_mprintf(
+      "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
+  );
 
-    /* Get the readers back into order. */
-    while( i-->0 ){
-      orderedDLReaderReorder(readers+i, nReaders-i);
-    }
+  if( !zCols || !zSql ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_declare_vtab(p->db, zSql);
   }
 
-  /* Copy over any remaining elements. */
-  if( nStart>0 ) rc = dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
-err:
-  dlwDestroy(&writer);
+  sqlite3_free(zSql);
+  sqlite3_free(zCols);
   return rc;
 }
 
-/* Helper function for posListUnion().  Compares the current position
-** between left and right, returning as standard C idiom of <0 if
-** left<right, >0 if left>right, and 0 if left==right.  "End" always
-** compares greater.
-*/
-static int posListCmp(PLReader *pLeft, PLReader *pRight){
-  assert( pLeft->iType==pRight->iType );
-  if( pLeft->iType==DL_DOCIDS ) return 0;
-
-  if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1;
-  if( plrAtEnd(pRight) ) return -1;
-
-  if( plrColumn(pLeft)<plrColumn(pRight) ) return -1;
-  if( plrColumn(pLeft)>plrColumn(pRight) ) return 1;
-
-  if( plrPosition(pLeft)<plrPosition(pRight) ) return -1;
-  if( plrPosition(pLeft)>plrPosition(pRight) ) return 1;
-  if( pLeft->iType==DL_POSITIONS ) return 0;
-
-  if( plrStartOffset(pLeft)<plrStartOffset(pRight) ) return -1;
-  if( plrStartOffset(pLeft)>plrStartOffset(pRight) ) return 1;
-
-  if( plrEndOffset(pLeft)<plrEndOffset(pRight) ) return -1;
-  if( plrEndOffset(pLeft)>plrEndOffset(pRight) ) return 1;
-
-  return 0;
-}
-
-/* Write the union of position lists in pLeft and pRight to pOut.
-** "Union" in this case meaning "All unique position tuples".  Should
-** work with any doclist type, though both inputs and the output
-** should be the same type.
-*/
-static int posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
-  PLReader left, right;
-  PLWriter writer;
-  int rc;
-
-  assert( dlrDocid(pLeft)==dlrDocid(pRight) );
-  assert( pLeft->iType==pRight->iType );
-  assert( pLeft->iType==pOut->iType );
-
-  rc = plrInit(&left, pLeft);
-  if( rc!=SQLITE_OK ) return rc;
-  rc = plrInit(&right, pRight);
-  if( rc!=SQLITE_OK ){
-    plrDestroy(&left);
-    return rc;
+/*
+** Create the backing store tables (%_content, %_segments and %_segdir)
+** required by the FTS3 table passed as the only argument. This is done
+** as part of the vtab xCreate() method.
+**
+** If the p->bHasDocsize boolean is true (indicating that this is an
+** FTS4 table, not an FTS3 table) then also create the %_docsize and
+** %_stat tables required by FTS4.
+*/
+static int fts3CreateTables(Fts3Table *p){
+  int rc = SQLITE_OK;             /* Return code */
+  int i;                          /* Iterator variable */
+  char *zContentCols;             /* Columns of %_content table */
+  sqlite3 *db = p->db;            /* The database connection */
+
+  /* Create a list of user columns for the content table */
+  if( p->bHasContent ){
+    zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
+    for(i=0; zContentCols && i<p->nColumn; i++){
+      char *z = p->azColumn[i];
+      zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
+    }
+    if( zContentCols==0 ) rc = SQLITE_NOMEM;
+
+    /* Create the content table */
+    fts3DbExec(&rc, db, 
+       "CREATE TABLE %Q.'%q_content'(%s)",
+       p->zDb, p->zName, zContentCols
+    );
+    sqlite3_free(zContentCols);
   }
-  plwInit(&writer, pOut, dlrDocid(pLeft));
-
-  while( !plrAtEnd(&left) || !plrAtEnd(&right) ){
-    int c = posListCmp(&left, &right);
-    if( c<0 ){
-      plwCopy(&writer, &left);
-      rc = plrStep(&left);
-      if( rc!=SQLITE_OK ) break;
-    }else if( c>0 ){
-      plwCopy(&writer, &right);
-      rc = plrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }else{
-      plwCopy(&writer, &left);
-      rc = plrStep(&left);
-      if( rc!=SQLITE_OK ) break;
-      rc = plrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }
+  /* Create other tables */
+  fts3DbExec(&rc, db, 
+      "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
+      p->zDb, p->zName
+  );
+  fts3DbExec(&rc, db, 
+      "CREATE TABLE %Q.'%q_segdir'("
+        "level INTEGER,"
+        "idx INTEGER,"
+        "start_block INTEGER,"
+        "leaves_end_block INTEGER,"
+        "end_block INTEGER,"
+        "root BLOB,"
+        "PRIMARY KEY(level, idx)"
+      ");",
+      p->zDb, p->zName
+  );
+  if( p->bHasDocsize ){
+    fts3DbExec(&rc, db, 
+        "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);",
+        p->zDb, p->zName
+    );
+    fts3DbExec(&rc, db, 
+        "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);",
+        p->zDb, p->zName
+    );
   }
-
-  plwTerminate(&writer);
-  plwDestroy(&writer);
-  plrDestroy(&left);
-  plrDestroy(&right);
   return rc;
 }
 
-/* Write the union of doclists in pLeft and pRight to pOut.  For
-** docids in common between the inputs, the union of the position
-** lists is written.  Inputs and outputs are always type DL_DEFAULT.
+/*
+** An sqlite3_exec() callback for fts3TableExists.
 */
-static int docListUnion(
-  const char *pLeft, int nLeft,
-  const char *pRight, int nRight,
-  DataBuffer *pOut      /* Write the combined doclist here */
-){
-  DLReader left, right;
-  DLWriter writer;
-  int rc;
-
-  if( nLeft==0 ){
-    if( nRight!=0) dataBufferAppend(pOut, pRight, nRight);
-    return SQLITE_OK;
-  }
-  if( nRight==0 ){
-    dataBufferAppend(pOut, pLeft, nLeft);
-    return SQLITE_OK;
-  }
-
-  rc = dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
-  if( rc!=SQLITE_OK ) return rc;
-  rc = dlrInit(&right, DL_DEFAULT, pRight, nRight);
-  if( rc!=SQLITE_OK){
-    dlrDestroy(&left);
-    return rc;
-  }
-  dlwInit(&writer, DL_DEFAULT, pOut);
-
-  while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
-    if( dlrAtEnd(&right) ){
-      rc = dlwCopy(&writer, &left);
-      if( rc!=SQLITE_OK) break;
-      rc = dlrStep(&left);
-      if( rc!=SQLITE_OK) break;
-    }else if( dlrAtEnd(&left) ){
-      rc = dlwCopy(&writer, &right);
-      if( rc!=SQLITE_OK ) break;
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }else if( dlrDocid(&left)<dlrDocid(&right) ){
-      rc = dlwCopy(&writer, &left);
-      if( rc!=SQLITE_OK ) break;
-      rc = dlrStep(&left);
-      if( rc!=SQLITE_OK ) break;
-    }else if( dlrDocid(&left)>dlrDocid(&right) ){
-      rc = dlwCopy(&writer, &right);
-      if( rc!=SQLITE_OK ) break;
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }else{
-      rc = posListUnion(&left, &right, &writer);
-      if( rc!=SQLITE_OK ) break;
-      rc = dlrStep(&left);
-      if( rc!=SQLITE_OK ) break;
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }
-  }
-
-  dlrDestroy(&left);
-  dlrDestroy(&right);
-  dlwDestroy(&writer);
-  return rc;
+static int fts3TableExistsCallback(void *pArg, int n, char **pp1, char **pp2){
+  UNUSED_PARAMETER(n);
+  UNUSED_PARAMETER(pp1);
+  UNUSED_PARAMETER(pp2);
+  *(int*)pArg = 1;
+  return 1;
 }
 
-/* 
-** This function is used as part of the implementation of phrase and
-** NEAR matching.
-**
-** pLeft and pRight are DLReaders positioned to the same docid in
-** lists of type DL_POSITION. This function writes an entry to the
-** DLWriter pOut for each position in pRight that is less than
-** (nNear+1) greater (but not equal to or smaller) than a position 
-** in pLeft. For example, if nNear is 0, and the positions contained
-** by pLeft and pRight are:
-**
-**    pLeft:  5 10 15 20
-**    pRight: 6  9 17 21
-**
-** then the docid is added to pOut. If pOut is of type DL_POSITIONS,
-** then a positionids "6" and "21" are also added to pOut.
-**
-** If boolean argument isSaveLeft is true, then positionids are copied
-** from pLeft instead of pRight. In the example above, the positions "5"
-** and "20" would be added instead of "6" and "21".
-*/
-static int posListPhraseMerge(
-  DLReader *pLeft, 
-  DLReader *pRight,
-  int nNear,
-  int isSaveLeft,
-  DLWriter *pOut
+/*
+** Determine if a table currently exists in the database.
+*/
+static void fts3TableExists(
+  int *pRc,             /* Success code */
+  sqlite3 *db,          /* The database connection to test */
+  const char *zDb,      /* ATTACHed database within the connection */
+  const char *zName,    /* Name of the FTS3 table */
+  const char *zSuffix,  /* Shadow table extension */
+  u8 *pResult           /* Write results here */
 ){
-  PLReader left, right;
-  PLWriter writer;
-  int match = 0;
-  int rc;
-
-  assert( dlrDocid(pLeft)==dlrDocid(pRight) );
-  assert( pOut->iType!=DL_POSITIONS_OFFSETS );
-
-  rc = plrInit(&left, pLeft);
-  if( rc!=SQLITE_OK ) return rc;
-  rc = plrInit(&right, pRight);
-  if( rc!=SQLITE_OK ){
-    plrDestroy(&left);
-    return rc;
-  }
-
-  while( !plrAtEnd(&left) && !plrAtEnd(&right) ){
-    if( plrColumn(&left)<plrColumn(&right) ){
-      rc = plrStep(&left);
-      if( rc!=SQLITE_OK ) break;
-    }else if( plrColumn(&left)>plrColumn(&right) ){
-      rc = plrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }else if( plrPosition(&left)>=plrPosition(&right) ){
-      rc = plrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }else{
-      if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){
-        if( !match ){
-          plwInit(&writer, pOut, dlrDocid(pLeft));
-          match = 1;
-        }
-        if( !isSaveLeft ){
-          plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
-        }else{
-          plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0);
-        }
-        rc = plrStep(&right);
-        if( rc!=SQLITE_OK ) break;
-      }else{
-        rc = plrStep(&left);
-        if( rc!=SQLITE_OK ) break;
-      }
-    }
-  }
-
-  if( match ){
-    plwTerminate(&writer);
-    plwDestroy(&writer);
-  }
-
-  plrDestroy(&left);
-  plrDestroy(&right);
-  return rc;
+  int rc = SQLITE_OK;
+  int res = 0;
+  char *zSql;
+  if( *pRc ) return;
+  zSql = sqlite3_mprintf(
+    "SELECT 1 FROM %Q.sqlite_master WHERE name='%q%s'",
+    zDb, zName, zSuffix
+  );    
+  rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0);
+  sqlite3_free(zSql);
+  *pResult = (u8)(res & 0xff);
+  if( rc!=SQLITE_ABORT ) *pRc = rc;
 }
 
 /*
-** Compare the values pointed to by the PLReaders passed as arguments. 
-** Return -1 if the value pointed to by pLeft is considered less than
-** the value pointed to by pRight, +1 if it is considered greater
-** than it, or 0 if it is equal. i.e.
-**
-**     (*pLeft - *pRight)
-**
-** A PLReader that is in the EOF condition is considered greater than
-** any other. If neither argument is in EOF state, the return value of
-** plrColumn() is used. If the plrColumn() values are equal, the
-** comparison is on the basis of plrPosition().
-*/
-static int plrCompare(PLReader *pLeft, PLReader *pRight){
-  assert(!plrAtEnd(pLeft) || !plrAtEnd(pRight));
-
-  if( plrAtEnd(pRight) || plrAtEnd(pLeft) ){
-    return (plrAtEnd(pRight) ? -1 : 1);
-  }
-  if( plrColumn(pLeft)!=plrColumn(pRight) ){
-    return ((plrColumn(pLeft)<plrColumn(pRight)) ? -1 : 1);
-  }
-  if( plrPosition(pLeft)!=plrPosition(pRight) ){
-    return ((plrPosition(pLeft)<plrPosition(pRight)) ? -1 : 1);
-  }
-  return 0;
-}
-
-/* We have two doclists with positions:  pLeft and pRight. Depending
-** on the value of the nNear parameter, perform either a phrase
-** intersection (if nNear==0) or a NEAR intersection (if nNear>0)
-** and write the results into pOut.
-**
-** A phrase intersection means that two documents only match
-** if pLeft.iPos+1==pRight.iPos.
-**
-** A NEAR intersection means that two documents only match if 
-** (abs(pLeft.iPos-pRight.iPos)<nNear).
-**
-** If a NEAR intersection is requested, then the nPhrase argument should
-** be passed the number of tokens in the two operands to the NEAR operator
-** combined. For example:
-**
-**       Query syntax               nPhrase
-**      ------------------------------------
-**       "A B C" NEAR "D E"         5
-**       A NEAR B                   2
-**
-** iType controls the type of data written to pOut.  If iType is
-** DL_POSITIONS, the positions are those from pRight.
-*/
-static int docListPhraseMerge(
-  const char *pLeft, int nLeft,
-  const char *pRight, int nRight,
-  int nNear,            /* 0 for a phrase merge, non-zero for a NEAR merge */
-  int nPhrase,          /* Number of tokens in left+right operands to NEAR */
-  DocListType iType,    /* Type of doclist to write to pOut */
-  DataBuffer *pOut      /* Write the combined doclist here */
+** This function is the implementation of both the xConnect and xCreate
+** methods of the FTS3 virtual table.
+**
+** The argv[] array contains the following:
+**
+**   argv[0]   -> module name  ("fts3" or "fts4")
+**   argv[1]   -> database name
+**   argv[2]   -> table name
+**   argv[...] -> "column name" and other module argument fields.
+*/
+static int fts3InitVtab(
+  int isCreate,                   /* True for xCreate, false for xConnect */
+  sqlite3 *db,                    /* The SQLite database connection */
+  void *pAux,                     /* Hash table containing tokenizers */
+  int argc,                       /* Number of elements in argv array */
+  const char * const *argv,       /* xCreate/xConnect argument array */
+  sqlite3_vtab **ppVTab,          /* Write the resulting vtab structure here */
+  char **pzErr                    /* Write any error message here */
 ){
-  DLReader left, right;
-  DLWriter writer;
-  int rc;
-
-  /* These two buffers are used in the 'while', but are declared here
-  ** to simplify error-handling.
-  */
-  DataBuffer one = {0, 0, 0};
-  DataBuffer two = {0, 0, 0};
-
-  if( nLeft==0 || nRight==0 ) return SQLITE_OK;
-
-  assert( iType!=DL_POSITIONS_OFFSETS );
-
-  rc = dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
-  if( rc!=SQLITE_OK ) return rc;
-  rc = dlrInit(&right, DL_POSITIONS, pRight, nRight);
-  if( rc!=SQLITE_OK ){
-    dlrDestroy(&left);
-    return rc;
-  }
-  dlwInit(&writer, iType, pOut);
-
-  while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
-    if( dlrDocid(&left)<dlrDocid(&right) ){
-      rc = dlrStep(&left);
-      if( rc!=SQLITE_OK ) goto err;
-    }else if( dlrDocid(&right)<dlrDocid(&left) ){
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) goto err;
-    }else{
-      if( nNear==0 ){
-        rc = posListPhraseMerge(&left, &right, 0, 0, &writer);
-        if( rc!=SQLITE_OK ) goto err;
-      }else{
-        /* This case occurs when two terms (simple terms or phrases) are
-         * connected by a NEAR operator, span (nNear+1). i.e.
-         *
-         *     '"terrible company" NEAR widget'
-         */
-        DLWriter dlwriter2;
-        DLReader dr1 = {0, 0, 0, 0, 0};
-        DLReader dr2 = {0, 0, 0, 0, 0};
-
-        dlwInit(&dlwriter2, iType, &one);
-        rc = posListPhraseMerge(&right, &left, nNear-3+nPhrase, 1, &dlwriter2);
-        if( rc!=SQLITE_OK ) goto err;
-        dlwInit(&dlwriter2, iType, &two);
-        rc = posListPhraseMerge(&left, &right, nNear-1, 0, &dlwriter2);
-        if( rc!=SQLITE_OK ) goto err;
-
-        if( one.nData){
-          rc = dlrInit(&dr1, iType, one.pData, one.nData);
-          if( rc!=SQLITE_OK ) goto err;
-        }
-        if( two.nData){
-          rc = dlrInit(&dr2, iType, two.pData, two.nData);
-          if( rc!=SQLITE_OK ) goto err;
-        }
-
-        if( !dlrAtEnd(&dr1) || !dlrAtEnd(&dr2) ){
-          PLReader pr1 = {0};
-          PLReader pr2 = {0};
-
-          PLWriter plwriter;
-          plwInit(&plwriter, &writer, dlrDocid(dlrAtEnd(&dr1)?&dr2:&dr1));
-
-          if( one.nData ){
-            rc = plrInit(&pr1, &dr1);
-            if( rc!=SQLITE_OK ) goto err;
-          }
-          if( two.nData ){
-            rc = plrInit(&pr2, &dr2);
-            if( rc!=SQLITE_OK ) goto err;
-          }
-          while( !plrAtEnd(&pr1) || !plrAtEnd(&pr2) ){
-            int iCompare = plrCompare(&pr1, &pr2);
-            switch( iCompare ){
-              case -1:
-                plwCopy(&plwriter, &pr1);
-                rc = plrStep(&pr1);
-                if( rc!=SQLITE_OK ) goto err;
-                break;
-              case 1:
-                plwCopy(&plwriter, &pr2);
-                rc = plrStep(&pr2);
-                if( rc!=SQLITE_OK ) goto err;
-                break;
-              case 0:
-                plwCopy(&plwriter, &pr1);
-                rc = plrStep(&pr1);
-                if( rc!=SQLITE_OK ) goto err;
-                rc = plrStep(&pr2);
-                if( rc!=SQLITE_OK ) goto err;
-                break;
-            }
-          }
-          plwTerminate(&plwriter);
-        }
-        dataBufferReset(&one);
-        dataBufferReset(&two);
-      }
-      rc = dlrStep(&left);
-      if( rc!=SQLITE_OK ) goto err;
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) goto err;
+  Fts3Hash *pHash = (Fts3Hash *)pAux;
+  Fts3Table *p;                   /* Pointer to allocated vtab */
+  int rc;                         /* Return code */
+  int i;                          /* Iterator variable */
+  int nByte;                      /* Size of allocation used for *p */
+  int iCol;                       /* Column index */
+  int nString = 0;                /* Bytes required to hold all column names */
+  int nCol = 0;                   /* Number of columns in the FTS table */
+  char *zCsr;                     /* Space for holding column names */
+  int nDb;                        /* Bytes required to hold database name */
+  int nName;                      /* Bytes required to hold table name */
+
+  const char *zTokenizer = 0;               /* Name of tokenizer to use */
+  sqlite3_tokenizer *pTokenizer = 0;        /* Tokenizer for this table */
+
+  nDb = (int)strlen(argv[1]) + 1;
+  nName = (int)strlen(argv[2]) + 1;
+  for(i=3; i<argc; i++){
+    char const *z = argv[i];
+    rc = sqlite3Fts3InitTokenizer(pHash, z, &pTokenizer, &zTokenizer, pzErr);
+    if( rc!=SQLITE_OK ){
+      return rc;
+    }
+    if( z!=zTokenizer ){
+      nString += (int)(strlen(z) + 1);
     }
   }
-
-err:
-  dataBufferDestroy(&one);
-  dataBufferDestroy(&two);
-  dlrDestroy(&left);
-  dlrDestroy(&right);
-  dlwDestroy(&writer);
-  return rc;
-}
-
-/* We have two DL_DOCIDS doclists:  pLeft and pRight.
-** Write the intersection of these two doclists into pOut as a
-** DL_DOCIDS doclist.
-*/
-static int docListAndMerge(
-  const char *pLeft, int nLeft,
-  const char *pRight, int nRight,
-  DataBuffer *pOut      /* Write the combined doclist here */
-){
-  DLReader left, right;
-  DLWriter writer;
-  int rc;
-
-  if( nLeft==0 || nRight==0 ) return SQLITE_OK;
-
-  rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
-  if( rc!=SQLITE_OK ) return rc;
-  rc = dlrInit(&right, DL_DOCIDS, pRight, nRight);
-  if( rc!=SQLITE_OK ){
-    dlrDestroy(&left);
-    return rc;
-  }
-  dlwInit(&writer, DL_DOCIDS, pOut);
-
-  while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
-    if( dlrDocid(&left)<dlrDocid(&right) ){
-      rc = dlrStep(&left);
-      if( rc!=SQLITE_OK ) break;
-    }else if( dlrDocid(&right)<dlrDocid(&left) ){
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }else{
-      dlwAdd(&writer, dlrDocid(&left));
-      rc = dlrStep(&left);
-      if( rc!=SQLITE_OK ) break;
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) break;
+  nCol = argc - 3 - (zTokenizer!=0);
+  if( zTokenizer==0 ){
+    rc = sqlite3Fts3InitTokenizer(pHash, 0, &pTokenizer, 0, pzErr);
+    if( rc!=SQLITE_OK ){
+      return rc;
     }
+    assert( pTokenizer );
   }
 
-  dlrDestroy(&left);
-  dlrDestroy(&right);
-  dlwDestroy(&writer);
-  return rc;
-}
-
-/* We have two DL_DOCIDS doclists:  pLeft and pRight.
-** Write the union of these two doclists into pOut as a
-** DL_DOCIDS doclist.
-*/
-static int docListOrMerge(
-  const char *pLeft, int nLeft,
-  const char *pRight, int nRight,
-  DataBuffer *pOut      /* Write the combined doclist here */
-){
-  DLReader left, right;
-  DLWriter writer;
-  int rc;
-
-  if( nLeft==0 ){
-    if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
-    return SQLITE_OK;
-  }
-  if( nRight==0 ){
-    dataBufferAppend(pOut, pLeft, nLeft);
-    return SQLITE_OK;
+  if( nCol==0 ){
+    nCol = 1;
   }
 
-  rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
-  if( rc!=SQLITE_OK ) return rc;
-  rc = dlrInit(&right, DL_DOCIDS, pRight, nRight);
-  if( rc!=SQLITE_OK ){
-    dlrDestroy(&left);
-    return rc;
-  }
-  dlwInit(&writer, DL_DOCIDS, pOut);
-
-  while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
-    if( dlrAtEnd(&right) ){
-      dlwAdd(&writer, dlrDocid(&left));
-      rc = dlrStep(&left);
-      if( rc!=SQLITE_OK ) break;
-    }else if( dlrAtEnd(&left) ){
-      dlwAdd(&writer, dlrDocid(&right));
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }else if( dlrDocid(&left)<dlrDocid(&right) ){
-      dlwAdd(&writer, dlrDocid(&left));
-      rc = dlrStep(&left);
-      if( rc!=SQLITE_OK ) break;
-    }else if( dlrDocid(&right)<dlrDocid(&left) ){
-      dlwAdd(&writer, dlrDocid(&right));
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }else{
-      dlwAdd(&writer, dlrDocid(&left));
-      rc = dlrStep(&left);
-      if( rc!=SQLITE_OK ) break;
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) break;
-    }
+  /* Allocate and populate the Fts3Table structure. */
+  nByte = sizeof(Fts3Table) +              /* Fts3Table */
+          nCol * sizeof(char *) +              /* azColumn */
+          nName +                              /* zName */
+          nDb +                                /* zDb */
+          nString;                             /* Space for azColumn strings */
+  p = (Fts3Table*)sqlite3_malloc(nByte);
+  if( p==0 ){
+    rc = SQLITE_NOMEM;
+    goto fts3_init_out;
+  }
+  memset(p, 0, nByte);
+
+  p->db = db;
+  p->nColumn = nCol;
+  p->nPendingData = 0;
+  p->azColumn = (char **)&p[1];
+  p->pTokenizer = pTokenizer;
+  p->nNodeSize = 1000;
+  p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
+  zCsr = (char *)&p->azColumn[nCol];
+
+  fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
+
+  /* Fill in the zName and zDb fields of the vtab structure. */
+  p->zName = zCsr;
+  memcpy(zCsr, argv[2], nName);
+  zCsr += nName;
+  p->zDb = zCsr;
+  memcpy(zCsr, argv[1], nDb);
+  zCsr += nDb;
+
+  /* Fill in the azColumn array */
+  iCol = 0;
+  for(i=3; i<argc; i++){
+    if( argv[i]!=zTokenizer ){
+      char *z; 
+      int n;
+      z = (char *)sqlite3Fts3NextToken(argv[i], &n);
+      memcpy(zCsr, z, n);
+      zCsr[n] = '\0';
+      sqlite3Fts3Dequote(zCsr);
+      p->azColumn[iCol++] = zCsr;
+      zCsr += n+1;
+      assert( zCsr <= &((char *)p)[nByte] );
+    }
+  }
+  if( iCol==0 ){
+    assert( nCol==1 );
+    p->azColumn[0] = "content";
+  }
+
+  /* If this is an xCreate call, create the underlying tables in the 
+  ** database. TODO: For xConnect(), it could verify that said tables exist.
+  */
+  if( isCreate ){
+    p->bHasContent = 1;
+    p->bHasDocsize = argv[0][3]=='4';
+    rc = fts3CreateTables(p);
+  }else{
+    rc = SQLITE_OK;
+    fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent);
+    fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize);
   }
+  if( rc!=SQLITE_OK ) goto fts3_init_out;
 
-  dlrDestroy(&left);
-  dlrDestroy(&right);
-  dlwDestroy(&writer);
-  return rc;
-}
-
-/* We have two DL_DOCIDS doclists:  pLeft and pRight.
-** Write into pOut as DL_DOCIDS doclist containing all documents that
-** occur in pLeft but not in pRight.
-*/
-static int docListExceptMerge(
-  const char *pLeft, int nLeft,
-  const char *pRight, int nRight,
-  DataBuffer *pOut      /* Write the combined doclist here */
-){
-  DLReader left, right;
-  DLWriter writer;
-  int rc;
+  rc = fts3DeclareVtab(p);
+  if( rc!=SQLITE_OK ) goto fts3_init_out;
 
-  if( nLeft==0 ) return SQLITE_OK;
-  if( nRight==0 ){
-    dataBufferAppend(pOut, pLeft, nLeft);
-    return SQLITE_OK;
-  }
+  *ppVTab = &p->base;
 
-  rc = dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
-  if( rc!=SQLITE_OK ) return rc;
-  rc = dlrInit(&right, DL_DOCIDS, pRight, nRight);
+fts3_init_out:
+  assert( p || (pTokenizer && rc!=SQLITE_OK) );
   if( rc!=SQLITE_OK ){
-    dlrDestroy(&left);
-    return rc;
-  }
-  dlwInit(&writer, DL_DOCIDS, pOut);
-
-  while( !dlrAtEnd(&left) ){
-    while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
-      rc = dlrStep(&right);
-      if( rc!=SQLITE_OK ) goto err;
-    }
-    if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){
-      dlwAdd(&writer, dlrDocid(&left));
+    if( p ){
+      fts3DisconnectMethod((sqlite3_vtab *)p);
+    }else{
+      pTokenizer->pModule->xDestroy(pTokenizer);
     }
-    rc = dlrStep(&left);
-    if( rc!=SQLITE_OK ) break;
   }
-
-err:
-  dlrDestroy(&left);
-  dlrDestroy(&right);
-  dlwDestroy(&writer);
   return rc;
 }
 
-static char *string_dup_n(const char *s, int n){
-  char *str = sqlite3_malloc(n + 1);
-  memcpy(str, s, n);
-  str[n] = '\0';
-  return str;
+/*
+** The xConnect() and xCreate() methods for the virtual table. All the
+** work is done in function fts3InitVtab().
+*/
+static int fts3ConnectMethod(
+  sqlite3 *db,                    /* Database connection */
+  void *pAux,                     /* Pointer to tokenizer hash table */
+  int argc,                       /* Number of elements in argv array */
+  const char * const *argv,       /* xCreate/xConnect argument array */
+  sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
+  char **pzErr                    /* OUT: sqlite3_malloc'd error message */
+){
+  return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
+}
+static int fts3CreateMethod(
+  sqlite3 *db,                    /* Database connection */
+  void *pAux,                     /* Pointer to tokenizer hash table */
+  int argc,                       /* Number of elements in argv array */
+  const char * const *argv,       /* xCreate/xConnect argument array */
+  sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
+  char **pzErr                    /* OUT: sqlite3_malloc'd error message */
+){
+  return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
 }
 
-/* Duplicate a string; the caller must free() the returned string.
- * (We don't use strdup() since it is not part of the standard C library and
- * may not be available everywhere.) */
-static char *string_dup(const char *s){
-  return string_dup_n(s, strlen(s));
-}
+/* 
+** Implementation of the xBestIndex method for FTS3 tables. There
+** are three possible strategies, in order of preference:
+**
+**   1. Direct lookup by rowid or docid. 
+**   2. Full-text search using a MATCH operator on a non-docid column.
+**   3. Linear scan of %_content table.
+*/
+static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
+  Fts3Table *p = (Fts3Table *)pVTab;
+  int i;                          /* Iterator variable */
+  int iCons = -1;                 /* Index of constraint to use */
 
-/* Format a string, replacing each occurrence of the % character with
- * zDb.zName.  This may be more convenient than sqlite_mprintf()
- * when one string is used repeatedly in a format string.
- * The caller must free() the returned string. */
-static char *string_format(const char *zFormat,
-                           const char *zDb, const char *zName){
-  const char *p;
-  size_t len = 0;
-  size_t nDb = strlen(zDb);
-  size_t nName = strlen(zName);
-  size_t nFullTableName = nDb+1+nName;
-  char *result;
-  char *r;
-
-  /* first compute length needed */
-  for(p = zFormat ; *p ; ++p){
-    len += (*p=='%' ? nFullTableName : 1);
-  }
-  len += 1;  /* for null terminator */
-
-  r = result = sqlite3_malloc(len);
-  for(p = zFormat; *p; ++p){
-    if( *p=='%' ){
-      memcpy(r, zDb, nDb);
-      r += nDb;
-      *r++ = '.';
-      memcpy(r, zName, nName);
-      r += nName;
-    } else {
-      *r++ = *p;
+  /* By default use a full table scan. This is an expensive option,
+  ** so search through the constraints to see if a more efficient 
+  ** strategy is possible.
+  */
+  pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
+  pInfo->estimatedCost = 500000;
+  for(i=0; i<pInfo->nConstraint; i++){
+    struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
+    if( pCons->usable==0 ) continue;
+
+    /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */
+    if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ 
+     && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 )
+    ){
+      pInfo->idxNum = FTS3_DOCID_SEARCH;
+      pInfo->estimatedCost = 1.0;
+      iCons = i;
+    }
+
+    /* A MATCH constraint. Use a full-text search.
+    **
+    ** If there is more than one MATCH constraint available, use the first
+    ** one encountered. If there is both a MATCH constraint and a direct
+    ** rowid/docid lookup, prefer the MATCH strategy. This is done even 
+    ** though the rowid/docid lookup is faster than a MATCH query, selecting
+    ** it would lead to an "unable to use function MATCH in the requested 
+    ** context" error.
+    */
+    if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH 
+     && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn
+    ){
+      pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn;
+      pInfo->estimatedCost = 2.0;
+      iCons = i;
+      break;
     }
   }
-  *r++ = '\0';
-  assert( r == result + len );
-  return result;
-}
 
-static int sql_exec(sqlite3 *db, const char *zDb, const char *zName,
-                    const char *zFormat){
-  char *zCommand = string_format(zFormat, zDb, zName);
-  int rc;
-  FTSTRACE(("FTS3 sql: %s\n", zCommand));
-  rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
-  sqlite3_free(zCommand);
-  return rc;
-}
-
-static int sql_prepare(sqlite3 *db, const char *zDb, const char *zName,
-                       sqlite3_stmt **ppStmt, const char *zFormat){
-  char *zCommand = string_format(zFormat, zDb, zName);
-  int rc;
-  FTSTRACE(("FTS3 prepare: %s\n", zCommand));
-  rc = sqlite3_prepare_v2(db, zCommand, -1, ppStmt, NULL);
-  sqlite3_free(zCommand);
-  return rc;
+  if( iCons>=0 ){
+    pInfo->aConstraintUsage[iCons].argvIndex = 1;
+    pInfo->aConstraintUsage[iCons].omit = 1;
+  } 
+  return SQLITE_OK;
 }
 
-/* end utility functions */
-
-/* Forward reference */
-typedef struct fulltext_vtab fulltext_vtab;
-
 /*
-** An instance of the following structure keeps track of generated
-** matching-word offset information and snippets.
-*/
-typedef struct Snippet {
-  int nMatch;     /* Total number of matches */
-  int nAlloc;     /* Space allocated for aMatch[] */
-  struct snippetMatch { /* One entry for each matching term */
-    char snStatus;       /* Status flag for use while constructing snippets */
-    short int iCol;      /* The column that contains the match */
-    short int iTerm;     /* The index in Query.pTerms[] of the matching term */
-    int iToken;          /* The index of the matching document token */
-    short int nByte;     /* Number of bytes in the term */
-    int iStart;          /* The offset to the first character of the term */
-  } *aMatch;      /* Points to space obtained from malloc */
-  char *zOffset;  /* Text rendering of aMatch[] */
-  int nOffset;    /* strlen(zOffset) */
-  char *zSnippet; /* Snippet text */
-  int nSnippet;   /* strlen(zSnippet) */
-} Snippet;
-
-
-typedef enum QueryType {
-  QUERY_GENERIC,   /* table scan */
-  QUERY_DOCID,     /* lookup by docid */
-  QUERY_FULLTEXT   /* QUERY_FULLTEXT + [i] is a full-text search for column i*/
-} QueryType;
-
-typedef enum fulltext_statement {
-  CONTENT_INSERT_STMT,
-  CONTENT_SELECT_STMT,
-  CONTENT_UPDATE_STMT,
-  CONTENT_DELETE_STMT,
-  CONTENT_EXISTS_STMT,
-
-  BLOCK_INSERT_STMT,
-  BLOCK_SELECT_STMT,
-  BLOCK_DELETE_STMT,
-  BLOCK_DELETE_ALL_STMT,
-
-  SEGDIR_MAX_INDEX_STMT,
-  SEGDIR_SET_STMT,
-  SEGDIR_SELECT_LEVEL_STMT,
-  SEGDIR_SPAN_STMT,
-  SEGDIR_DELETE_STMT,
-  SEGDIR_SELECT_SEGMENT_STMT,
-  SEGDIR_SELECT_ALL_STMT,
-  SEGDIR_DELETE_ALL_STMT,
-  SEGDIR_COUNT_STMT,
-
-  MAX_STMT                     /* Always at end! */
-} fulltext_statement;
-
-/* These must exactly match the enum above. */
-/* TODO(shess): Is there some risk that a statement will be used in two
-** cursors at once, e.g.  if a query joins a virtual table to itself?
-** If so perhaps we should move some of these to the cursor object.
+** Implementation of xOpen method.
 */
-static const char *const fulltext_zStatement[MAX_STMT] = {
-  /* CONTENT_INSERT */ NULL,  /* generated in contentInsertStatement() */
-  /* CONTENT_SELECT */ NULL,  /* generated in contentSelectStatement() */
-  /* CONTENT_UPDATE */ NULL,  /* generated in contentUpdateStatement() */
-  /* CONTENT_DELETE */ "delete from %_content where docid = ?",
-  /* CONTENT_EXISTS */ "select docid from %_content limit 1",
-
-  /* BLOCK_INSERT */
-  "insert into %_segments (blockid, block) values (null, ?)",
-  /* BLOCK_SELECT */ "select block from %_segments where blockid = ?",
-  /* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?",
-  /* BLOCK_DELETE_ALL */ "delete from %_segments",
-
-  /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
-  /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
-  /* SEGDIR_SELECT_LEVEL */
-  "select start_block, leaves_end_block, root, idx from %_segdir "
-  " where level = ? order by idx",
-  /* SEGDIR_SPAN */
-  "select min(start_block), max(end_block) from %_segdir "
-  " where level = ? and start_block <> 0",
-  /* SEGDIR_DELETE */ "delete from %_segdir where level = ?",
-
-  /* NOTE(shess): The first three results of the following two
-  ** statements must match.
-  */
-  /* SEGDIR_SELECT_SEGMENT */
-  "select start_block, leaves_end_block, root from %_segdir "
-  " where level = ? and idx = ?",
-  /* SEGDIR_SELECT_ALL */
-  "select start_block, leaves_end_block, root from %_segdir "
-  " order by level desc, idx asc",
-  /* SEGDIR_DELETE_ALL */ "delete from %_segdir",
-  /* SEGDIR_COUNT */ "select count(*), ifnull(max(level),0) from %_segdir",
-};
+static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+  sqlite3_vtab_cursor *pCsr;               /* Allocated cursor */
 
-/*
-** A connection to a fulltext index is an instance of the following
-** structure.  The xCreate and xConnect methods create an instance
-** of this structure and xDestroy and xDisconnect free that instance.
-** All other methods receive a pointer to the structure as one of their
-** arguments.
-*/
-struct fulltext_vtab {
-  sqlite3_vtab base;               /* Base class used by SQLite core */
-  sqlite3 *db;                     /* The database connection */
-  const char *zDb;                 /* logical database name */
-  const char *zName;               /* virtual table name */
-  int nColumn;                     /* number of columns in virtual table */
-  char **azColumn;                 /* column names.  malloced */
-  char **azContentColumn;          /* column names in content table; malloced */
-  sqlite3_tokenizer *pTokenizer;   /* tokenizer for inserts and queries */
-
-  /* Precompiled statements which we keep as long as the table is
-  ** open.
-  */
-  sqlite3_stmt *pFulltextStatements[MAX_STMT];
+  UNUSED_PARAMETER(pVTab);
 
-  /* Precompiled statements used for segment merges.  We run a
-  ** separate select across the leaf level of each tree being merged.
-  */
-  sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
-  /* The statement used to prepare pLeafSelectStmts. */
-#define LEAF_SELECT \
-  "select block from %_segments where blockid between ? and ? order by blockid"
-
-  /* These buffer pending index updates during transactions.
-  ** nPendingData estimates the memory size of the pending data.  It
-  ** doesn't include the hash-bucket overhead, nor any malloc
-  ** overhead.  When nPendingData exceeds kPendingThreshold, the
-  ** buffer is flushed even before the transaction closes.
-  ** pendingTerms stores the data, and is only valid when nPendingData
-  ** is >=0 (nPendingData<0 means pendingTerms has not been
-  ** initialized).  iPrevDocid is the last docid written, used to make
-  ** certain we're inserting in sorted order.
+  /* Allocate a buffer large enough for an Fts3Cursor structure. If the
+  ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, 
+  ** if the allocation fails, return SQLITE_NOMEM.
   */
-  int nPendingData;
-#define kPendingThreshold (1*1024*1024)
-  sqlite_int64 iPrevDocid;
-  fts3Hash pendingTerms;
-};
-
-/*
-** When the core wants to do a query, it create a cursor using a
-** call to xOpen.  This structure is an instance of a cursor.  It
-** is destroyed by xClose.
-*/
-typedef struct fulltext_cursor {
-  sqlite3_vtab_cursor base;        /* Base class used by SQLite core */
-  QueryType iCursorType;           /* Copy of sqlite3_index_info.idxNum */
-  sqlite3_stmt *pStmt;             /* Prepared statement in use by the cursor */
-  int eof;                         /* True if at End Of Results */
-  Fts3Expr *pExpr;                 /* Parsed MATCH query string */
-  Snippet snippet;                 /* Cached snippet for the current row */
-  int iColumn;                     /* Column being searched */
-  DataBuffer result;               /* Doclist results from fulltextQuery */
-  DLReader reader;                 /* Result reader if result not empty */
-} fulltext_cursor;
-
-static fulltext_vtab *cursor_vtab(fulltext_cursor *c){
-  return (fulltext_vtab *) c->base.pVtab;
-}
-
-static const sqlite3_module fts3Module;   /* forward declaration */
-
-/* Return a dynamically generated statement of the form
- *   insert into %_content (docid, ...) values (?, ...)
- */
-static const char *contentInsertStatement(fulltext_vtab *v){
-  StringBuffer sb;
-  int i;
-
-  initStringBuffer(&sb);
-  append(&sb, "insert into %_content (docid, ");
-  appendList(&sb, v->nColumn, v->azContentColumn);
-  append(&sb, ") values (?");
-  for(i=0; i<v->nColumn; ++i)
-    append(&sb, ", ?");
-  append(&sb, ")");
-  return stringBufferData(&sb);
-}
-
-/* Return a dynamically generated statement of the form
- *   select <content columns> from %_content where docid = ?
- */
-static const char *contentSelectStatement(fulltext_vtab *v){
-  StringBuffer sb;
-  initStringBuffer(&sb);
-  append(&sb, "SELECT ");
-  appendList(&sb, v->nColumn, v->azContentColumn);
-  append(&sb, " FROM %_content WHERE docid = ?");
-  return stringBufferData(&sb);
-}
-
-/* Return a dynamically generated statement of the form
- *   update %_content set [col_0] = ?, [col_1] = ?, ...
- *                    where docid = ?
- */
-static const char *contentUpdateStatement(fulltext_vtab *v){
-  StringBuffer sb;
-  int i;
-
-  initStringBuffer(&sb);
-  append(&sb, "update %_content set ");
-  for(i=0; i<v->nColumn; ++i) {
-    if( i>0 ){
-      append(&sb, ", ");
-    }
-    append(&sb, v->azContentColumn[i]);
-    append(&sb, " = ?");
-  }
-  append(&sb, " where docid = ?");
-  return stringBufferData(&sb);
-}
-
-/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
-** If the indicated statement has never been prepared, it is prepared
-** and cached, otherwise the cached version is reset.
-*/
-static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
-                             sqlite3_stmt **ppStmt){
-  assert( iStmt<MAX_STMT );
-  if( v->pFulltextStatements[iStmt]==NULL ){
-    const char *zStmt;
-    int rc;
-    switch( iStmt ){
-      case CONTENT_INSERT_STMT:
-        zStmt = contentInsertStatement(v); break;
-      case CONTENT_SELECT_STMT:
-        zStmt = contentSelectStatement(v); break;
-      case CONTENT_UPDATE_STMT:
-        zStmt = contentUpdateStatement(v); break;
-      default:
-        zStmt = fulltext_zStatement[iStmt];
-    }
-    rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt],
-                         zStmt);
-    if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt);
-    if( rc!=SQLITE_OK ) return rc;
-  } else {
-    int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
-    if( rc!=SQLITE_OK ) return rc;
+  *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor));
+  if( !pCsr ){
+    return SQLITE_NOMEM;
   }
-
-  *ppStmt = v->pFulltextStatements[iStmt];
+  memset(pCsr, 0, sizeof(Fts3Cursor));
   return SQLITE_OK;
 }
 
-/* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and
-** SQLITE_ROW to SQLITE_ERROR.  Useful for statements like UPDATE,
-** where we expect no results.
-*/
-static int sql_single_step(sqlite3_stmt *s){
-  int rc = sqlite3_step(s);
-  return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
-}
-
-/* Like sql_get_statement(), but for special replicated LEAF_SELECT
-** statements.  idx -1 is a special case for an uncached version of
-** the statement (used in the optimize implementation).
-*/
-/* TODO(shess) Write version for generic statements and then share
-** that between the cached-statement functions.
+/*
+** Close the cursor.  For additional information see the documentation
+** on the xClose method of the virtual table interface.
 */
-static int sql_get_leaf_statement(fulltext_vtab *v, int idx,
-                                  sqlite3_stmt **ppStmt){
-  assert( idx>=-1 && idx<MERGE_COUNT );
-  if( idx==-1 ){
-    return sql_prepare(v->db, v->zDb, v->zName, ppStmt, LEAF_SELECT);
-  }else if( v->pLeafSelectStmts[idx]==NULL ){
-    int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx],
-                         LEAF_SELECT);
-    if( rc!=SQLITE_OK ) return rc;
-  }else{
-    int rc = sqlite3_reset(v->pLeafSelectStmts[idx]);
-    if( rc!=SQLITE_OK ) return rc;
-  }
-
-  *ppStmt = v->pLeafSelectStmts[idx];
+static int fulltextClose(sqlite3_vtab_cursor *pCursor){
+  Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+  sqlite3_finalize(pCsr->pStmt);
+  sqlite3Fts3ExprFree(pCsr->pExpr);
+  sqlite3_free(pCsr->aDoclist);
+  sqlite3_free(pCsr->aMatchinfo);
+  sqlite3_free(pCsr);
   return SQLITE_OK;
 }
 
-/* insert into %_content (docid, ...) values ([docid], [pValues])
-** If the docid contains SQL NULL, then a unique docid will be
-** generated.
-*/
-static int content_insert(fulltext_vtab *v, sqlite3_value *docid,
-                          sqlite3_value **pValues){
-  sqlite3_stmt *s;
-  int i;
-  int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_value(s, 1, docid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  for(i=0; i<v->nColumn; ++i){
-    rc = sqlite3_bind_value(s, 2+i, pValues[i]);
-    if( rc!=SQLITE_OK ) return rc;
-  }
-
-  return sql_single_step(s);
-}
-
-/* update %_content set col0 = pValues[0], col1 = pValues[1], ...
- *                  where docid = [iDocid] */
-static int content_update(fulltext_vtab *v, sqlite3_value **pValues,
-                          sqlite_int64 iDocid){
-  sqlite3_stmt *s;
-  int i;
-  int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  for(i=0; i<v->nColumn; ++i){
-    rc = sqlite3_bind_value(s, 1+i, pValues[i]);
-    if( rc!=SQLITE_OK ) return rc;
-  }
-
-  rc = sqlite3_bind_int64(s, 1+v->nColumn, iDocid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step(s);
-}
-
-static void freeStringArray(int nString, const char **pString){
-  int i;
-
-  for (i=0 ; i < nString ; ++i) {
-    if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]);
-  }
-  sqlite3_free((void *) pString);
-}
-
-/* select * from %_content where docid = [iDocid]
- * The caller must delete the returned array and all strings in it.
- * null fields will be NULL in the returned array.
- *
- * TODO: Perhaps we should return pointer/length strings here for consistency
- * with other code which uses pointer/length. */
-static int content_select(fulltext_vtab *v, sqlite_int64 iDocid,
-                          const char ***pValues){
-  sqlite3_stmt *s;
-  const char **values;
-  int i;
-  int rc;
-
-  *pValues = NULL;
-
-  rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 1, iDocid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_step(s);
-  if( rc!=SQLITE_ROW ) return rc;
-
-  values = (const char **) sqlite3_malloc(v->nColumn * sizeof(const char *));
-  for(i=0; i<v->nColumn; ++i){
-    if( sqlite3_column_type(s, i)==SQLITE_NULL ){
-      values[i] = NULL;
+/*
+** Position the pCsr->pStmt statement so that it is on the row
+** of the %_content table that contains the last match.  Return
+** SQLITE_OK on success.  
+*/
+static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
+  if( pCsr->isRequireSeek ){
+    pCsr->isRequireSeek = 0;
+    sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
+    if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
+      return SQLITE_OK;
     }else{
-      values[i] = string_dup((char*)sqlite3_column_text(s, i));
+      int rc = sqlite3_reset(pCsr->pStmt);
+      if( rc==SQLITE_OK ){
+        /* If no row was found and no error has occured, then the %_content
+        ** table is missing a row that is present in the full-text index.
+        ** The data structures are corrupt.
+        */
+        rc = SQLITE_CORRUPT;
+      }
+      pCsr->isEof = 1;
+      if( pContext ){
+        sqlite3_result_error_code(pContext, rc);
+      }
+      return rc;
     }
-  }
-
-  /* We expect only one row.  We must execute another sqlite3_step()
-   * to complete the iteration; otherwise the table will remain locked. */
-  rc = sqlite3_step(s);
-  if( rc==SQLITE_DONE ){
-    *pValues = values;
+  }else{
     return SQLITE_OK;
   }
-
-  freeStringArray(v->nColumn, values);
-  return rc;
 }
 
-/* delete from %_content where docid = [iDocid ] */
-static int content_delete(fulltext_vtab *v, sqlite_int64 iDocid){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 1, iDocid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step(s);
-}
-
-/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if
-** no rows exist, and any error in case of failure.
-*/
-static int content_exists(fulltext_vtab *v){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_step(s);
-  if( rc!=SQLITE_ROW ) return rc;
-
-  /* We expect only one row.  We must execute another sqlite3_step()
-   * to complete the iteration; otherwise the table will remain locked. */
-  rc = sqlite3_step(s);
-  if( rc==SQLITE_DONE ) return SQLITE_ROW;
-  if( rc==SQLITE_ROW ) return SQLITE_ERROR;
-  return rc;
-}
-
-/* insert into %_segments values ([pData])
-**   returns assigned blockid in *piBlockid
-*/
-static int block_insert(fulltext_vtab *v, const char *pData, int nData,
-                        sqlite_int64 *piBlockid){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_step(s);
-  if( rc==SQLITE_ROW ) return SQLITE_ERROR;
-  if( rc!=SQLITE_DONE ) return rc;
-
-  /* blockid column is an alias for rowid. */
-  *piBlockid = sqlite3_last_insert_rowid(v->db);
-  return SQLITE_OK;
-}
-
-/* delete from %_segments
-**   where blockid between [iStartBlockid] and [iEndBlockid]
-**
-** Deletes the range of blocks, inclusive, used to delete the blocks
-** which form a segment.
-*/
-static int block_delete(fulltext_vtab *v,
-                        sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 1, iStartBlockid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 2, iEndBlockid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step(s);
-}
-
-/* Returns SQLITE_ROW with *pidx set to the maximum segment idx found
-** at iLevel.  Returns SQLITE_DONE if there are no segments at
-** iLevel.  Otherwise returns an error.
-*/
-static int segdir_max_index(fulltext_vtab *v, int iLevel, int *pidx){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int(s, 1, iLevel);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_step(s);
-  /* Should always get at least one row due to how max() works. */
-  if( rc==SQLITE_DONE ) return SQLITE_DONE;
-  if( rc!=SQLITE_ROW ) return rc;
-
-  /* NULL means that there were no inputs to max(). */
-  if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
-    rc = sqlite3_step(s);
-    if( rc==SQLITE_ROW ) return SQLITE_ERROR;
-    return rc;
-  }
-
-  *pidx = sqlite3_column_int(s, 0);
-
-  /* We expect only one row.  We must execute another sqlite3_step()
-   * to complete the iteration; otherwise the table will remain locked. */
-  rc = sqlite3_step(s);
-  if( rc==SQLITE_ROW ) return SQLITE_ERROR;
-  if( rc!=SQLITE_DONE ) return rc;
-  return SQLITE_ROW;
-}
-
-/* insert into %_segdir values (
-**   [iLevel], [idx],
-**   [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid],
-**   [pRootData]
-** )
-*/
-static int segdir_set(fulltext_vtab *v, int iLevel, int idx,
-                      sqlite_int64 iStartBlockid,
-                      sqlite_int64 iLeavesEndBlockid,
-                      sqlite_int64 iEndBlockid,
-                      const char *pRootData, int nRootData){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int(s, 1, iLevel);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int(s, 2, idx);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 3, iStartBlockid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 5, iEndBlockid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step(s);
-}
-
-/* Queries %_segdir for the block span of the segments in level
-** iLevel.  Returns SQLITE_DONE if there are no blocks for iLevel,
-** SQLITE_ROW if there are blocks, else an error.
-*/
-static int segdir_span(fulltext_vtab *v, int iLevel,
-                       sqlite_int64 *piStartBlockid,
-                       sqlite_int64 *piEndBlockid){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int(s, 1, iLevel);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_step(s);
-  if( rc==SQLITE_DONE ) return SQLITE_DONE;  /* Should never happen */
-  if( rc!=SQLITE_ROW ) return rc;
-
-  /* This happens if all segments at this level are entirely inline. */
-  if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
-    /* We expect only one row.  We must execute another sqlite3_step()
-     * to complete the iteration; otherwise the table will remain locked. */
-    int rc2 = sqlite3_step(s);
-    if( rc2==SQLITE_ROW ) return SQLITE_ERROR;
-    return rc2;
+/*
+** Advance the cursor to the next row in the %_content table that
+** matches the search criteria.  For a MATCH search, this will be
+** the next row that matches.  For a full-table scan, this will be
+** simply the next row in the %_content table.  For a docid lookup,
+** this routine simply sets the EOF flag.
+**
+** Return SQLITE_OK if nothing goes wrong.  SQLITE_OK is returned
+** even if we reach end-of-file.  The fts3EofMethod() will be called
+** subsequently to determine whether or not an EOF was hit.
+*/
+static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
+  int rc = SQLITE_OK;             /* Return code */
+  Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+
+  if( pCsr->aDoclist==0 ){
+    if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
+      pCsr->isEof = 1;
+      rc = sqlite3_reset(pCsr->pStmt);
+    }
+  }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
+    pCsr->isEof = 1;
+  }else{
+    sqlite3_reset(pCsr->pStmt);
+    fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
+    pCsr->isRequireSeek = 1;
+    pCsr->isMatchinfoNeeded = 1;
   }
-
-  *piStartBlockid = sqlite3_column_int64(s, 0);
-  *piEndBlockid = sqlite3_column_int64(s, 1);
-
-  /* We expect only one row.  We must execute another sqlite3_step()
-   * to complete the iteration; otherwise the table will remain locked. */
-  rc = sqlite3_step(s);
-  if( rc==SQLITE_ROW ) return SQLITE_ERROR;
-  if( rc!=SQLITE_DONE ) return rc;
-  return SQLITE_ROW;
+  return rc;
 }
 
-/* Delete the segment blocks and segment directory records for all
-** segments at iLevel.
-*/
-static int segdir_delete(fulltext_vtab *v, int iLevel){
-  sqlite3_stmt *s;
-  sqlite_int64 iStartBlockid, iEndBlockid;
-  int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid);
-  if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
-
-  if( rc==SQLITE_ROW ){
-    rc = block_delete(v, iStartBlockid, iEndBlockid);
-    if( rc!=SQLITE_OK ) return rc;
-  }
-
-  /* Delete the segment directory itself. */
-  rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 1, iLevel);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step(s);
-}
 
-/* Delete entire fts index, SQLITE_OK on success, relevant error on
-** failure.
-*/
-static int segdir_delete_all(fulltext_vtab *v){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sql_single_step(s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return sql_single_step(s);
-}
+/*
+** The buffer pointed to by argument zNode (size nNode bytes) contains the
+** root node of a b-tree segment. The segment is guaranteed to be at least
+** one level high (i.e. the root node is not also a leaf). If successful,
+** this function locates the leaf node of the segment that may contain the 
+** term specified by arguments zTerm and nTerm and writes its block number 
+** to *piLeaf.
+**
+** It is possible that the returned leaf node does not contain the specified
+** term. However, if the segment does contain said term, it is stored on
+** the identified leaf node. Because this function only inspects interior
+** segment nodes (and never loads leaf nodes into memory), it is not possible
+** to be sure.
+**
+** If an error occurs, an error code other than SQLITE_OK is returned.
+*/ 
+static int fts3SelectLeaf(
+  Fts3Table *p,                   /* Virtual table handle */
+  const char *zTerm,              /* Term to select leaves for */
+  int nTerm,                      /* Size of term zTerm in bytes */
+  const char *zNode,              /* Buffer containing segment interior node */
+  int nNode,                      /* Size of buffer at zNode */
+  sqlite3_int64 *piLeaf           /* Selected leaf node */
+){
+  int rc = SQLITE_OK;             /* Return code */
+  const char *zCsr = zNode;       /* Cursor to iterate through node */
+  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
+  char *zBuffer = 0;              /* Buffer to load terms into */
+  int nAlloc = 0;                 /* Size of allocated buffer */
+
+  while( 1 ){
+    int isFirstTerm = 1;          /* True when processing first term on page */
+    int iHeight;                  /* Height of this node in tree */
+    sqlite3_int64 iChild;         /* Block id of child node to descend to */
+    int nBlock;                   /* Size of child node in bytes */
+
+    zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
+    zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+  
+    while( zCsr<zEnd ){
+      int cmp;                    /* memcmp() result */
+      int nSuffix;                /* Size of term suffix */
+      int nPrefix = 0;            /* Size of term prefix */
+      int nBuffer;                /* Total term size */
+  
+      /* Load the next term on the node into zBuffer */
+      if( !isFirstTerm ){
+        zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
+      }
+      isFirstTerm = 0;
+      zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
+      if( nPrefix+nSuffix>nAlloc ){
+        char *zNew;
+        nAlloc = (nPrefix+nSuffix) * 2;
+        zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+        if( !zNew ){
+          sqlite3_free(zBuffer);
+          return SQLITE_NOMEM;
+        }
+        zBuffer = zNew;
+      }
+      memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
+      nBuffer = nPrefix + nSuffix;
+      zCsr += nSuffix;
+  
+      /* Compare the term we are searching for with the term just loaded from
+      ** the interior node. If the specified term is greater than or equal
+      ** to the term from the interior node, then all terms on the sub-tree 
+      ** headed by node iChild are smaller than zTerm. No need to search 
+      ** iChild.
+      **
+      ** If the interior node term is larger than the specified term, then
+      ** the tree headed by iChild may contain the specified term.
+      */
+      cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
+      if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break;
+      iChild++;
+    };
 
-/* Returns SQLITE_OK with *pnSegments set to the number of entries in
-** %_segdir and *piMaxLevel set to the highest level which has a
-** segment.  Otherwise returns the SQLite error which caused failure.
-*/
-static int segdir_count(fulltext_vtab *v, int *pnSegments, int *piMaxLevel){
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
+    /* If (iHeight==1), the children of this interior node are leaves. The
+    ** specified term may be present on leaf node iChild.
+    */
+    if( iHeight==1 ){
+      *piLeaf = iChild;
+      break;
+    }
 
-  rc = sqlite3_step(s);
-  /* TODO(shess): This case should not be possible?  Should stronger
-  ** measures be taken if it happens?
-  */
-  if( rc==SQLITE_DONE ){
-    *pnSegments = 0;
-    *piMaxLevel = 0;
-    return SQLITE_OK;
+    /* Descend to interior node iChild. */
+    rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
+    if( rc!=SQLITE_OK ) break;
+    zEnd = &zCsr[nBlock];
   }
-  if( rc!=SQLITE_ROW ) return rc;
-
-  *pnSegments = sqlite3_column_int(s, 0);
-  *piMaxLevel = sqlite3_column_int(s, 1);
-
-  /* We expect only one row.  We must execute another sqlite3_step()
-   * to complete the iteration; otherwise the table will remain locked. */
-  rc = sqlite3_step(s);
-  if( rc==SQLITE_DONE ) return SQLITE_OK;
-  if( rc==SQLITE_ROW ) return SQLITE_ERROR;
+  sqlite3_free(zBuffer);
   return rc;
 }
 
-/* TODO(shess) clearPendingTerms() is far down the file because
-** writeZeroSegment() is far down the file because LeafWriter is far
-** down the file.  Consider refactoring the code to move the non-vtab
-** code above the vtab code so that we don't need this forward
-** reference.
-*/
-static int clearPendingTerms(fulltext_vtab *v);
-
 /*
-** Free the memory used to contain a fulltext_vtab structure.
+** This function is used to create delta-encoded serialized lists of FTS3 
+** varints. Each call to this function appends a single varint to a list.
 */
-static void fulltext_vtab_destroy(fulltext_vtab *v){
-  int iStmt, i;
-
-  FTSTRACE(("FTS3 Destroy %p\n", v));
-  for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
-    if( v->pFulltextStatements[iStmt]!=NULL ){
-      sqlite3_finalize(v->pFulltextStatements[iStmt]);
-      v->pFulltextStatements[iStmt] = NULL;
-    }
-  }
-
-  for( i=0; i<MERGE_COUNT; i++ ){
-    if( v->pLeafSelectStmts[i]!=NULL ){
-      sqlite3_finalize(v->pLeafSelectStmts[i]);
-      v->pLeafSelectStmts[i] = NULL;
-    }
-  }
+static void fts3PutDeltaVarint(
+  char **pp,                      /* IN/OUT: Output pointer */
+  sqlite3_int64 *piPrev,          /* IN/OUT: Previous value written to list */
+  sqlite3_int64 iVal              /* Write this value to the list */
+){
+  assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) );
+  *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev);
+  *piPrev = iVal;
+}
 
-  if( v->pTokenizer!=NULL ){
-    v->pTokenizer->pModule->xDestroy(v->pTokenizer);
-    v->pTokenizer = NULL;
+/*
+** When this function is called, *ppPoslist is assumed to point to the 
+** start of a position-list. After it returns, *ppPoslist points to the
+** first byte after the position-list.
+**
+** A position list is list of positions (delta encoded) and columns for 
+** a single document record of a doclist.  So, in other words, this
+** routine advances *ppPoslist so that it points to the next docid in
+** the doclist, or to the first byte past the end of the doclist.
+**
+** If pp is not NULL, then the contents of the position list are copied
+** to *pp. *pp is set to point to the first byte past the last byte copied
+** before this function returns.
+*/
+static void fts3PoslistCopy(char **pp, char **ppPoslist){
+  char *pEnd = *ppPoslist;
+  char c = 0;
+
+  /* The end of a position list is marked by a zero encoded as an FTS3 
+  ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by
+  ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail
+  ** of some other, multi-byte, value.
+  **
+  ** The following while-loop moves pEnd to point to the first byte that is not 
+  ** immediately preceded by a byte with the 0x80 bit set. Then increments
+  ** pEnd once more so that it points to the byte immediately following the
+  ** last byte in the position-list.
+  */
+  while( *pEnd | c ){
+    c = *pEnd++ & 0x80;
+    testcase( c!=0 && (*pEnd)==0 );
   }
+  pEnd++;  /* Advance past the POS_END terminator byte */
 
-  clearPendingTerms(v);
-
-  sqlite3_free(v->azColumn);
-  for(i = 0; i < v->nColumn; ++i) {
-    sqlite3_free(v->azContentColumn[i]);
+  if( pp ){
+    int n = (int)(pEnd - *ppPoslist);
+    char *p = *pp;
+    memcpy(p, *ppPoslist, n);
+    p += n;
+    *pp = p;
   }
-  sqlite3_free(v->azContentColumn);
-  sqlite3_free(v);
+  *ppPoslist = pEnd;
 }
 
 /*
-** Token types for parsing the arguments to xConnect or xCreate.
-*/
-#define TOKEN_EOF         0    /* End of file */
-#define TOKEN_SPACE       1    /* Any kind of whitespace */
-#define TOKEN_ID          2    /* An identifier */
-#define TOKEN_STRING      3    /* A string literal */
-#define TOKEN_PUNCT       4    /* A single punctuation character */
-
-/*
-** If X is a character that can be used in an identifier then
-** ftsIdChar(X) will be true.  Otherwise it is false.
+** When this function is called, *ppPoslist is assumed to point to the 
+** start of a column-list. After it returns, *ppPoslist points to the
+** to the terminator (POS_COLUMN or POS_END) byte of the column-list.
+**
+** A column-list is list of delta-encoded positions for a single column
+** within a single document within a doclist.
 **
-** For ASCII, any character with the high-order bit set is
-** allowed in an identifier.  For 7-bit characters, 
-** isFtsIdChar[X] must be 1.
+** The column-list is terminated either by a POS_COLUMN varint (1) or
+** a POS_END varint (0).  This routine leaves *ppPoslist pointing to
+** the POS_COLUMN or POS_END that terminates the column-list.
 **
-** Ticket #1066.  the SQL standard does not allow '$' in the
-** middle of identfiers.  But many SQL implementations do. 
-** SQLite will allow '$' in identifiers for compatibility.
-** But the feature is undocumented.
+** If pp is not NULL, then the contents of the column-list are copied
+** to *pp. *pp is set to point to the first byte past the last byte copied
+** before this function returns.  The POS_COLUMN or POS_END terminator
+** is not copied into *pp.
 */
-static const char isFtsIdChar[] = {
-/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
-    0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
-    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
-    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
-};
-#define ftsIdChar(C)  (((c=C)&0x80)!=0 || (c>0x1f && isFtsIdChar[c-0x20]))
-
+static void fts3ColumnlistCopy(char **pp, char **ppPoslist){
+  char *pEnd = *ppPoslist;
+  char c = 0;
 
-/*
-** Return the length of the token that begins at z[0]. 
-** Store the token type in *tokenType before returning.
-*/
-static int ftsGetToken(const char *z, int *tokenType){
-  int i, c;
-  switch( *z ){
-    case 0: {
-      *tokenType = TOKEN_EOF;
-      return 0;
-    }
-    case ' ': case '\t': case '\n': case '\f': case '\r': {
-      for(i=1; safe_isspace(z[i]); i++){}
-      *tokenType = TOKEN_SPACE;
-      return i;
-    }
-    case '`':
-    case '\'':
-    case '"': {
-      int delim = z[0];
-      for(i=1; (c=z[i])!=0; i++){
-        if( c==delim ){
-          if( z[i+1]==delim ){
-            i++;
-          }else{
-            break;
-          }
-        }
-      }
-      *tokenType = TOKEN_STRING;
-      return i + (c!=0);
-    }
-    case '[': {
-      for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
-      *tokenType = TOKEN_ID;
-      return i;
-    }
-    default: {
-      if( !ftsIdChar(*z) ){
-        break;
-      }
-      for(i=1; ftsIdChar(z[i]); i++){}
-      *tokenType = TOKEN_ID;
-      return i;
-    }
+  /* A column-list is terminated by either a 0x01 or 0x00 byte that is
+  ** not part of a multi-byte varint.
+  */
+  while( 0xFE & (*pEnd | c) ){
+    c = *pEnd++ & 0x80;
+    testcase( c!=0 && ((*pEnd)&0xfe)==0 );
   }
-  *tokenType = TOKEN_PUNCT;
-  return 1;
+  if( pp ){
+    int n = (int)(pEnd - *ppPoslist);
+    char *p = *pp;
+    memcpy(p, *ppPoslist, n);
+    p += n;
+    *pp = p;
+  }
+  *ppPoslist = pEnd;
 }
 
 /*
-** A token extracted from a string is an instance of the following
-** structure.
+** Value used to signify the end of an position-list. This is safe because
+** it is not possible to have a document with 2^31 terms.
 */
-typedef struct FtsToken {
-  const char *z;       /* Pointer to token text.  Not '\000' terminated */
-  short int n;         /* Length of the token text in bytes. */
-} FtsToken;
+#define POSITION_LIST_END 0x7fffffff
 
 /*
-** Given a input string (which is really one of the argv[] parameters
-** passed into xConnect or xCreate) split the string up into tokens.
-** Return an array of pointers to '\000' terminated strings, one string
-** for each non-whitespace token.
-**
-** The returned array is terminated by a single NULL pointer.
-**
-** Space to hold the returned array is obtained from a single
-** malloc and should be freed by passing the return value to free().
-** The individual strings within the token list are all a part of
-** the single memory allocation and will all be freed at once.
-*/
-static char **tokenizeString(const char *z, int *pnToken){
-  int nToken = 0;
-  FtsToken *aToken = sqlite3_malloc( strlen(z) * sizeof(aToken[0]) );
-  int n = 1;
-  int e, i;
-  int totalSize = 0;
-  char **azToken;
-  char *zCopy;
-  while( n>0 ){
-    n = ftsGetToken(z, &e);
-    if( e!=TOKEN_SPACE ){
-      aToken[nToken].z = z;
-      aToken[nToken].n = n;
-      nToken++;
-      totalSize += n+1;
-    }
-    z += n;
-  }
-  azToken = (char**)sqlite3_malloc( nToken*sizeof(char*) + totalSize );
-  zCopy = (char*)&azToken[nToken];
-  nToken--;
-  for(i=0; i<nToken; i++){
-    azToken[i] = zCopy;
-    n = aToken[i].n;
-    memcpy(zCopy, aToken[i].z, n);
-    zCopy[n] = 0;
-    zCopy += n+1;
+** This function is used to help parse position-lists. When this function is
+** called, *pp may point to the start of the next varint in the position-list
+** being parsed, or it may point to 1 byte past the end of the position-list
+** (in which case **pp will be a terminator bytes POS_END (0) or
+** (1)).
+**
+** If *pp points past the end of the current position-list, set *pi to 
+** POSITION_LIST_END and return. Otherwise, read the next varint from *pp,
+** increment the current value of *pi by the value read, and set *pp to
+** point to the next value before returning.
+**
+** Before calling this routine *pi must be initialized to the value of
+** the previous position, or zero if we are reading the first position
+** in the position-list.  Because positions are delta-encoded, the value
+** of the previous position is needed in order to compute the value of
+** the next position.
+*/
+static void fts3ReadNextPos(
+  char **pp,                    /* IN/OUT: Pointer into position-list buffer */
+  sqlite3_int64 *pi             /* IN/OUT: Value read from position-list */
+){
+  if( (**pp)&0xFE ){
+    fts3GetDeltaVarint(pp, pi);
+    *pi -= 2;
+  }else{
+    *pi = POSITION_LIST_END;
   }
-  azToken[nToken] = 0;
-  sqlite3_free(aToken);
-  *pnToken = nToken;
-  return azToken;
 }
 
 /*
-** Convert an SQL-style quoted string into a normal string by removing
-** the quote characters.  The conversion is done in-place.  If the
-** input does not begin with a quote character, then this routine
-** is a no-op.
-**
-** Examples:
+** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by
+** the value of iCol encoded as a varint to *pp.   This will start a new
+** column list.
 **
-**     "abc"   becomes   abc
-**     'xyz'   becomes   xyz
-**     [pqr]   becomes   pqr
-**     `mno`   becomes   mno
+** Set *pp to point to the byte just after the last byte written before 
+** returning (do not modify it if iCol==0). Return the total number of bytes
+** written (0 if iCol==0).
 */
-static void dequoteString(char *z){
-  int quote;
-  int i, j;
-  if( z==0 ) return;
-  quote = z[0];
-  switch( quote ){
-    case '\'':  break;
-    case '"':   break;
-    case '`':   break;                /* For MySQL compatibility */
-    case '[':   quote = ']';  break;  /* For MS SqlServer compatibility */
-    default:    return;
-  }
-  for(i=1, j=0; z[i]; i++){
-    if( z[i]==quote ){
-      if( z[i+1]==quote ){
-        z[j++] = quote;
-        i++;
-      }else{
-        z[j++] = 0;
-        break;
-      }
-    }else{
-      z[j++] = z[i];
-    }
+static int fts3PutColNumber(char **pp, int iCol){
+  int n = 0;                      /* Number of bytes written */
+  if( iCol ){
+    char *p = *pp;                /* Output pointer */
+    n = 1 + sqlite3Fts3PutVarint(&p[1], iCol);
+    *p = 0x01;
+    *pp = &p[n];
   }
+  return n;
 }
 
 /*
-** The input azIn is a NULL-terminated list of tokens.  Remove the first
-** token and all punctuation tokens.  Remove the quotes from
-** around string literal tokens.
-**
-** Example:
-**
-**     input:      tokenize chinese ( 'simplifed' , 'mixed' )
-**     output:     chinese simplifed mixed
-**
-** Another example:
-**
-**     input:      delimiters ( '[' , ']' , '...' )
-**     output:     [ ] ...
-*/
-static void tokenListToIdList(char **azIn){
-  int i, j;
-  if( azIn ){
-    for(i=0, j=-1; azIn[i]; i++){
-      if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){
-        dequoteString(azIn[i]);
-        if( j>=0 ){
-          azIn[j] = azIn[i];
+** Compute the union of two position lists.  The output written
+** into *pp contains all positions of both *pp1 and *pp2 in sorted
+** order and with any duplicates removed.  All pointers are
+** updated appropriately.   The caller is responsible for insuring
+** that there is enough space in *pp to hold the complete output.
+*/
+static void fts3PoslistMerge(
+  char **pp,                      /* Output buffer */
+  char **pp1,                     /* Left input list */
+  char **pp2                      /* Right input list */
+){
+  char *p = *pp;
+  char *p1 = *pp1;
+  char *p2 = *pp2;
+
+  while( *p1 || *p2 ){
+    int iCol1;         /* The current column index in pp1 */
+    int iCol2;         /* The current column index in pp2 */
+
+    if( *p1==POS_COLUMN ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);
+    else if( *p1==POS_END ) iCol1 = POSITION_LIST_END;
+    else iCol1 = 0;
+
+    if( *p2==POS_COLUMN ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);
+    else if( *p2==POS_END ) iCol2 = POSITION_LIST_END;
+    else iCol2 = 0;
+
+    if( iCol1==iCol2 ){
+      sqlite3_int64 i1 = 0;       /* Last position from pp1 */
+      sqlite3_int64 i2 = 0;       /* Last position from pp2 */
+      sqlite3_int64 iPrev = 0;
+      int n = fts3PutColNumber(&p, iCol1);
+      p1 += n;
+      p2 += n;
+
+      /* At this point, both p1 and p2 point to the start of column-lists
+      ** for the same column (the column with index iCol1 and iCol2).
+      ** A column-list is a list of non-negative delta-encoded varints, each 
+      ** incremented by 2 before being stored. Each list is terminated by a
+      ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists
+      ** and writes the results to buffer p. p is left pointing to the byte
+      ** after the list written. No terminator (POS_END or POS_COLUMN) is
+      ** written to the output.
+      */
+      fts3GetDeltaVarint(&p1, &i1);
+      fts3GetDeltaVarint(&p2, &i2);
+      do {
+        fts3PutDeltaVarint(&p, &iPrev, (i1<i2) ? i1 : i2); 
+        iPrev -= 2;
+        if( i1==i2 ){
+          fts3ReadNextPos(&p1, &i1);
+          fts3ReadNextPos(&p2, &i2);
+        }else if( i1<i2 ){
+          fts3ReadNextPos(&p1, &i1);
+        }else{
+          fts3ReadNextPos(&p2, &i2);
         }
-        j++;
-      }
-    }
-    azIn[j] = 0;
-  }
-}
-
-
-/*
-** Find the first alphanumeric token in the string zIn.  Null-terminate
-** this token.  Remove any quotation marks.  And return a pointer to
-** the result.
-*/
-static char *firstToken(char *zIn, char **pzTail){
-  int n, ttype;
-  while(1){
-    n = ftsGetToken(zIn, &ttype);
-    if( ttype==TOKEN_SPACE ){
-      zIn += n;
-    }else if( ttype==TOKEN_EOF ){
-      *pzTail = zIn;
-      return 0;
+      }while( i1!=POSITION_LIST_END || i2!=POSITION_LIST_END );
+    }else if( iCol1<iCol2 ){
+      p1 += fts3PutColNumber(&p, iCol1);
+      fts3ColumnlistCopy(&p, &p1);
     }else{
-      zIn[n] = 0;
-      *pzTail = &zIn[1];
-      dequoteString(zIn);
-      return zIn;
+      p2 += fts3PutColNumber(&p, iCol2);
+      fts3ColumnlistCopy(&p, &p2);
     }
   }
-  /*NOTREACHED*/
-}
 
-/* Return true if...
-**
-**   *  s begins with the string t, ignoring case
-**   *  s is longer than t
-**   *  The first character of s beyond t is not a alphanumeric
-** 
-** Ignore leading space in *s.
-**
-** To put it another way, return true if the first token of
-** s[] is t[].
-*/
-static int startsWith(const char *s, const char *t){
-  while( safe_isspace(*s) ){ s++; }
-  while( *t ){
-    if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0;
-  }
-  return *s!='_' && !safe_isalnum(*s);
+  *p++ = POS_END;
+  *pp = p;
+  *pp1 = p1 + 1;
+  *pp2 = p2 + 1;
 }
 
 /*
-** An instance of this structure defines the "spec" of a
-** full text index.  This structure is populated by parseSpec
-** and use by fulltextConnect and fulltextCreate.
-*/
-typedef struct TableSpec {
-  const char *zDb;         /* Logical database name */
-  const char *zName;       /* Name of the full-text index */
-  int nColumn;             /* Number of columns to be indexed */
-  char **azColumn;         /* Original names of columns to be indexed */
-  char **azContentColumn;  /* Column names for %_content */
-  char **azTokenizer;      /* Name of tokenizer and its arguments */
-} TableSpec;
+** nToken==1 searches for adjacent positions.
+*/
+static int fts3PoslistPhraseMerge(
+  char **pp,                      /* Output buffer */
+  int nToken,                     /* Maximum difference in token positions */
+  int isSaveLeft,                 /* Save the left position */
+  char **pp1,                     /* Left input list */
+  char **pp2                      /* Right input list */
+){
+  char *p = (pp ? *pp : 0);
+  char *p1 = *pp1;
+  char *p2 = *pp2;
+
+  int iCol1 = 0;
+  int iCol2 = 0;
+  assert( *p1!=0 && *p2!=0 );
+  if( *p1==POS_COLUMN ){ 
+    p1++;
+    p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+  }
+  if( *p2==POS_COLUMN ){ 
+    p2++;
+    p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
+  }
+
+  while( 1 ){
+    if( iCol1==iCol2 ){
+      char *pSave = p;
+      sqlite3_int64 iPrev = 0;
+      sqlite3_int64 iPos1 = 0;
+      sqlite3_int64 iPos2 = 0;
+
+      if( pp && iCol1 ){
+        *p++ = POS_COLUMN;
+        p += sqlite3Fts3PutVarint(p, iCol1);
+      }
 
-/*
-** Reclaim all of the memory used by a TableSpec
-*/
-static void clearTableSpec(TableSpec *p) {
-  sqlite3_free(p->azColumn);
-  sqlite3_free(p->azContentColumn);
-  sqlite3_free(p->azTokenizer);
-}
+      assert( *p1!=POS_END && *p1!=POS_COLUMN );
+      assert( *p2!=POS_END && *p2!=POS_COLUMN );
+      fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
+      fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
 
-/* Parse a CREATE VIRTUAL TABLE statement, which looks like this:
- *
- * CREATE VIRTUAL TABLE email
- *        USING fts3(subject, body, tokenize mytokenizer(myarg))
- *
- * We return parsed information in a TableSpec structure.
- * 
- */
-static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv,
-                     char**pzErr){
-  int i, n;
-  char *z, *zDummy;
-  char **azArg;
-  const char *zTokenizer = 0;    /* argv[] entry describing the tokenizer */
-
-  assert( argc>=3 );
-  /* Current interface:
-  ** argv[0] - module name
-  ** argv[1] - database name
-  ** argv[2] - table name
-  ** argv[3..] - columns, optionally followed by tokenizer specification
-  **             and snippet delimiters specification.
-  */
+      while( 1 ){
+        if( iPos2>iPos1 && iPos2<=iPos1+nToken ){
+          sqlite3_int64 iSave;
+          if( !pp ){
+            fts3PoslistCopy(0, &p2);
+            fts3PoslistCopy(0, &p1);
+            *pp1 = p1;
+            *pp2 = p2;
+            return 1;
+          }
+          iSave = isSaveLeft ? iPos1 : iPos2;
+          fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
+          pSave = 0;
+        }
+        if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){
+          if( (*p2&0xFE)==0 ) break;
+          fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
+        }else{
+          if( (*p1&0xFE)==0 ) break;
+          fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
+        }
+      }
 
-  /* Make a copy of the complete argv[][] array in a single allocation.
-  ** The argv[][] array is read-only and transient.  We can write to the
-  ** copy in order to modify things and the copy is persistent.
-  */
-  CLEAR(pSpec);
-  for(i=n=0; i<argc; i++){
-    n += strlen(argv[i]) + 1;
-  }
-  azArg = sqlite3_malloc( sizeof(char*)*argc + n );
-  if( azArg==0 ){
-    return SQLITE_NOMEM;
-  }
-  z = (char*)&azArg[argc];
-  for(i=0; i<argc; i++){
-    azArg[i] = z;
-    strcpy(z, argv[i]);
-    z += strlen(z)+1;
-  }
+      if( pSave ){
+        assert( pp && p );
+        p = pSave;
+      }
 
-  /* Identify the column names and the tokenizer and delimiter arguments
-  ** in the argv[][] array.
-  */
-  pSpec->zDb = azArg[1];
-  pSpec->zName = azArg[2];
-  pSpec->nColumn = 0;
-  pSpec->azColumn = azArg;
-  zTokenizer = "tokenize simple";
-  for(i=3; i<argc; ++i){
-    if( startsWith(azArg[i],"tokenize") ){
-      zTokenizer = azArg[i];
-    }else{
-      z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy);
-      pSpec->nColumn++;
+      fts3ColumnlistCopy(0, &p1);
+      fts3ColumnlistCopy(0, &p2);
+      assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 );
+      if( 0==*p1 || 0==*p2 ) break;
+
+      p1++;
+      p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+      p2++;
+      p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
     }
-  }
-  if( pSpec->nColumn==0 ){
-    azArg[0] = "content";
-    pSpec->nColumn = 1;
-  }
 
-  /*
-  ** Construct the list of content column names.
-  **
-  ** Each content column name will be of the form cNNAAAA
-  ** where NN is the column number and AAAA is the sanitized
-  ** column name.  "sanitized" means that special characters are
-  ** converted to "_".  The cNN prefix guarantees that all column
-  ** names are unique.
-  **
-  ** The AAAA suffix is not strictly necessary.  It is included
-  ** for the convenience of people who might examine the generated
-  ** %_content table and wonder what the columns are used for.
-  */
-  pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) );
-  if( pSpec->azContentColumn==0 ){
-    clearTableSpec(pSpec);
-    return SQLITE_NOMEM;
-  }
-  for(i=0; i<pSpec->nColumn; i++){
-    char *p;
-    pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
-    for (p = pSpec->azContentColumn[i]; *p ; ++p) {
-      if( !safe_isalnum(*p) ) *p = '_';
+    /* Advance pointer p1 or p2 (whichever corresponds to the smaller of
+    ** iCol1 and iCol2) so that it points to either the 0x00 that marks the
+    ** end of the position list, or the 0x01 that precedes the next 
+    ** column-number in the position list. 
+    */
+    else if( iCol1<iCol2 ){
+      fts3ColumnlistCopy(0, &p1);
+      if( 0==*p1 ) break;
+      p1++;
+      p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+    }else{
+      fts3ColumnlistCopy(0, &p2);
+      if( 0==*p2 ) break;
+      p2++;
+      p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
     }
   }
 
-  /*
-  ** Parse the tokenizer specification string.
-  */
-  pSpec->azTokenizer = tokenizeString(zTokenizer, &n);
-  tokenListToIdList(pSpec->azTokenizer);
-
-  return SQLITE_OK;
+  fts3PoslistCopy(0, &p2);
+  fts3PoslistCopy(0, &p1);
+  *pp1 = p1;
+  *pp2 = p2;
+  if( !pp || *pp==p ){
+    return 0;
+  }
+  *p++ = 0x00;
+  *pp = p;
+  return 1;
 }
 
 /*
-** Generate a CREATE TABLE statement that describes the schema of
-** the virtual table.  Return a pointer to this schema string.
-**
-** Space is obtained from sqlite3_mprintf() and should be freed
-** using sqlite3_free().
-*/
-static char *fulltextSchema(
-  int nColumn,                  /* Number of columns */
-  const char *const* azColumn,  /* List of columns */
-  const char *zTableName        /* Name of the table */
+** Merge two position-lists as required by the NEAR operator.
+*/
+static int fts3PoslistNearMerge(
+  char **pp,                      /* Output buffer */
+  char *aTmp,                     /* Temporary buffer space */
+  int nRight,                     /* Maximum difference in token positions */
+  int nLeft,                      /* Maximum difference in token positions */
+  char **pp1,                     /* IN/OUT: Left input list */
+  char **pp2                      /* IN/OUT: Right input list */
 ){
-  int i;
-  char *zSchema, *zNext;
-  const char *zSep = "(";
-  zSchema = sqlite3_mprintf("CREATE TABLE x");
-  for(i=0; i<nColumn; i++){
-    zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]);
-    sqlite3_free(zSchema);
-    zSchema = zNext;
-    zSep = ",";
+  char *p1 = *pp1;
+  char *p2 = *pp2;
+
+  if( !pp ){
+    if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1;
+    *pp1 = p1;
+    *pp2 = p2;
+    return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1);
+  }else{
+    char *pTmp1 = aTmp;
+    char *pTmp2;
+    char *aTmp2;
+    int res = 1;
+
+    fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2);
+    aTmp2 = pTmp2 = pTmp1;
+    *pp1 = p1;
+    *pp2 = p2;
+    fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1);
+    if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
+      fts3PoslistMerge(pp, &aTmp, &aTmp2);
+    }else if( pTmp1!=aTmp ){
+      fts3PoslistCopy(pp, &aTmp);
+    }else if( pTmp2!=aTmp2 ){
+      fts3PoslistCopy(pp, &aTmp2);
+    }else{
+      res = 0;
+    }
+
+    return res;
   }
-  zNext = sqlite3_mprintf("%s,%Q HIDDEN", zSchema, zTableName);
-  sqlite3_free(zSchema);
-  zSchema = zNext;
-  zNext = sqlite3_mprintf("%s,docid HIDDEN)", zSchema);
-  sqlite3_free(zSchema);
-  return zNext;
 }
 
 /*
-** Build a new sqlite3_vtab structure that will describe the
-** fulltext index defined by spec.
+** Values that may be used as the first parameter to fts3DoclistMerge().
 */
-static int constructVtab(
-  sqlite3 *db,              /* The SQLite database connection */
-  fts3Hash *pHash,          /* Hash table containing tokenizers */
-  TableSpec *spec,          /* Parsed spec information from parseSpec() */
-  sqlite3_vtab **ppVTab,    /* Write the resulting vtab structure here */
-  char **pzErr              /* Write any error message here */
-){
-  int rc;
-  int n;
-  fulltext_vtab *v = 0;
-  const sqlite3_tokenizer_module *m = NULL;
-  char *schema;
-
-  char const *zTok;         /* Name of tokenizer to use for this fts table */
-  int nTok;                 /* Length of zTok, including nul terminator */
-
-  v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab));
-  if( v==0 ) return SQLITE_NOMEM;
-  CLEAR(v);
-  /* sqlite will initialize v->base */
-  v->db = db;
-  v->zDb = spec->zDb;       /* Freed when azColumn is freed */
-  v->zName = spec->zName;   /* Freed when azColumn is freed */
-  v->nColumn = spec->nColumn;
-  v->azContentColumn = spec->azContentColumn;
-  spec->azContentColumn = 0;
-  v->azColumn = spec->azColumn;
-  spec->azColumn = 0;
-
-  if( spec->azTokenizer==0 ){
-    return SQLITE_NOMEM;
-  }
-
-  zTok = spec->azTokenizer[0]; 
-  if( !zTok ){
-    zTok = "simple";
-  }
-  nTok = strlen(zTok)+1;
+#define MERGE_NOT        2        /* D + D -> D */
+#define MERGE_AND        3        /* D + D -> D */
+#define MERGE_OR         4        /* D + D -> D */
+#define MERGE_POS_OR     5        /* P + P -> P */
+#define MERGE_PHRASE     6        /* P + P -> D */
+#define MERGE_POS_PHRASE 7        /* P + P -> P */
+#define MERGE_NEAR       8        /* P + P -> D */
+#define MERGE_POS_NEAR   9        /* P + P -> P */
 
-  m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zTok, nTok);
-  if( !m ){
-    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
-    rc = SQLITE_ERROR;
-    goto err;
-  }
+/*
+** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
+** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
+** which is guaranteed to be large enough to hold the results. The number
+** of bytes written to aBuffer is stored in *pnBuffer before returning.
+**
+** If successful, SQLITE_OK is returned. Otherwise, if a malloc error
+** occurs while allocating a temporary buffer as part of the merge operation,
+** SQLITE_NOMEM is returned.
+*/
+static int fts3DoclistMerge(
+  int mergetype,                  /* One of the MERGE_XXX constants */
+  int nParam1,                    /* Used by MERGE_NEAR and MERGE_POS_NEAR */
+  int nParam2,                    /* Used by MERGE_NEAR and MERGE_POS_NEAR */
+  char *aBuffer,                  /* Pre-allocated output buffer */
+  int *pnBuffer,                  /* OUT: Bytes written to aBuffer */
+  char *a1,                       /* Buffer containing first doclist */
+  int n1,                         /* Size of buffer a1 */
+  char *a2,                       /* Buffer containing second doclist */
+  int n2                          /* Size of buffer a2 */
+){
+  sqlite3_int64 i1 = 0;
+  sqlite3_int64 i2 = 0;
+  sqlite3_int64 iPrev = 0;
+
+  char *p = aBuffer;
+  char *p1 = a1;
+  char *p2 = a2;
+  char *pEnd1 = &a1[n1];
+  char *pEnd2 = &a2[n2];
+
+  assert( mergetype==MERGE_OR     || mergetype==MERGE_POS_OR 
+       || mergetype==MERGE_AND    || mergetype==MERGE_NOT
+       || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
+       || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
+  );
 
-  for(n=0; spec->azTokenizer[n]; n++){}
-  if( n ){
-    rc = m->xCreate(n-1, (const char*const*)&spec->azTokenizer[1],
-                    &v->pTokenizer);
-  }else{
-    rc = m->xCreate(0, 0, &v->pTokenizer);
+  if( !aBuffer ){
+    *pnBuffer = 0;
+    return SQLITE_NOMEM;
   }
-  if( rc!=SQLITE_OK ) goto err;
-  v->pTokenizer->pModule = m;
-
-  /* TODO: verify the existence of backing tables foo_content, foo_term */
-
-  schema = fulltextSchema(v->nColumn, (const char*const*)v->azColumn,
-                          spec->zName);
-  rc = sqlite3_declare_vtab(db, schema);
-  sqlite3_free(schema);
-  if( rc!=SQLITE_OK ) goto err;
 
-  memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
-
-  /* Indicate that the buffer is not live. */
-  v->nPendingData = -1;
-
-  *ppVTab = &v->base;
-  FTSTRACE(("FTS3 Connect %p\n", v));
-
-  return rc;
+  /* Read the first docid from each doclist */
+  fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+  fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+
+  switch( mergetype ){
+    case MERGE_OR:
+    case MERGE_POS_OR:
+      while( p1 || p2 ){
+        if( p2 && p1 && i1==i2 ){
+          fts3PutDeltaVarint(&p, &iPrev, i1);
+          if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2);
+          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+        }else if( !p2 || (p1 && i1<i2) ){
+          fts3PutDeltaVarint(&p, &iPrev, i1);
+          if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1);
+          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+        }else{
+          fts3PutDeltaVarint(&p, &iPrev, i2);
+          if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2);
+          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+        }
+      }
+      break;
 
-err:
-  fulltext_vtab_destroy(v);
-  return rc;
-}
+    case MERGE_AND:
+      while( p1 && p2 ){
+        if( i1==i2 ){
+          fts3PutDeltaVarint(&p, &iPrev, i1);
+          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+        }else if( i1<i2 ){
+          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+        }else{
+          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+        }
+      }
+      break;
 
-static int fulltextConnect(
-  sqlite3 *db,
-  void *pAux,
-  int argc, const char *const*argv,
-  sqlite3_vtab **ppVTab,
-  char **pzErr
-){
-  TableSpec spec;
-  int rc = parseSpec(&spec, argc, argv, pzErr);
-  if( rc!=SQLITE_OK ) return rc;
+    case MERGE_NOT:
+      while( p1 ){
+        if( p2 && i1==i2 ){
+          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+        }else if( !p2 || i1<i2 ){
+          fts3PutDeltaVarint(&p, &iPrev, i1);
+          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+        }else{
+          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+        }
+      }
+      break;
 
-  rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr);
-  clearTableSpec(&spec);
-  return rc;
-}
+    case MERGE_POS_PHRASE:
+    case MERGE_PHRASE: {
+      char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
+      while( p1 && p2 ){
+        if( i1==i2 ){
+          char *pSave = p;
+          sqlite3_int64 iPrevSave = iPrev;
+          fts3PutDeltaVarint(&p, &iPrev, i1);
+          if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){
+            p = pSave;
+            iPrev = iPrevSave;
+          }
+          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+        }else if( i1<i2 ){
+          fts3PoslistCopy(0, &p1);
+          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+        }else{
+          fts3PoslistCopy(0, &p2);
+          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+        }
+      }
+      break;
+    }
 
-/* The %_content table holds the text of each document, with
-** the docid column exposed as the SQLite rowid for the table.
-*/
-/* TODO(shess) This comment needs elaboration to match the updated
-** code.  Work it into the top-of-file comment at that time.
-*/
-static int fulltextCreate(sqlite3 *db, void *pAux,
-                          int argc, const char * const *argv,
-                          sqlite3_vtab **ppVTab, char **pzErr){
-  int rc;
-  TableSpec spec;
-  StringBuffer schema;
-  FTSTRACE(("FTS3 Create\n"));
+    default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); {
+      char *aTmp = 0;
+      char **ppPos = 0;
 
-  rc = parseSpec(&spec, argc, argv, pzErr);
-  if( rc!=SQLITE_OK ) return rc;
+      if( mergetype==MERGE_POS_NEAR ){
+        ppPos = &p;
+        aTmp = sqlite3_malloc(2*(n1+n2+1));
+        if( !aTmp ){
+          return SQLITE_NOMEM;
+        }
+      }
 
-  initStringBuffer(&schema);
-  append(&schema, "CREATE TABLE %_content(");
-  append(&schema, "  docid INTEGER PRIMARY KEY,");
-  appendList(&schema, spec.nColumn, spec.azContentColumn);
-  append(&schema, ")");
-  rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema));
-  stringBufferDestroy(&schema);
-  if( rc!=SQLITE_OK ) goto out;
-
-  rc = sql_exec(db, spec.zDb, spec.zName,
-                "create table %_segments("
-                "  blockid INTEGER PRIMARY KEY,"
-                "  block blob"
-                ");"
-                );
-  if( rc!=SQLITE_OK ) goto out;
-
-  rc = sql_exec(db, spec.zDb, spec.zName,
-                "create table %_segdir("
-                "  level integer,"
-                "  idx integer,"
-                "  start_block integer,"
-                "  leaves_end_block integer,"
-                "  end_block integer,"
-                "  root blob,"
-                "  primary key(level, idx)"
-                ");");
-  if( rc!=SQLITE_OK ) goto out;
-
-  rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr);
-
-out:
-  clearTableSpec(&spec);
-  return rc;
-}
+      while( p1 && p2 ){
+        if( i1==i2 ){
+          char *pSave = p;
+          sqlite3_int64 iPrevSave = iPrev;
+          fts3PutDeltaVarint(&p, &iPrev, i1);
 
-/* Decide how to handle an SQL query. */
-static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
-  fulltext_vtab *v = (fulltext_vtab *)pVTab;
-  int i;
-  FTSTRACE(("FTS3 BestIndex\n"));
-
-  for(i=0; i<pInfo->nConstraint; ++i){
-    const struct sqlite3_index_constraint *pConstraint;
-    pConstraint = &pInfo->aConstraint[i];
-    if( pConstraint->usable ) {
-      if( (pConstraint->iColumn==-1 || pConstraint->iColumn==v->nColumn+1) &&
-          pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
-        pInfo->idxNum = QUERY_DOCID;      /* lookup by docid */
-        FTSTRACE(("FTS3 QUERY_DOCID\n"));
-      } else if( pConstraint->iColumn>=0 && pConstraint->iColumn<=v->nColumn &&
-                 pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
-        /* full-text search */
-        pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn;
-        FTSTRACE(("FTS3 QUERY_FULLTEXT %d\n", pConstraint->iColumn));
-      } else continue;
-
-      pInfo->aConstraintUsage[i].argvIndex = 1;
-      pInfo->aConstraintUsage[i].omit = 1;
-
-      /* An arbitrary value for now.
-       * TODO: Perhaps docid matches should be considered cheaper than
-       * full-text searches. */
-      pInfo->estimatedCost = 1.0;   
+          if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){
+            iPrev = iPrevSave;
+            p = pSave;
+          }
 
-      return SQLITE_OK;
+          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+        }else if( i1<i2 ){
+          fts3PoslistCopy(0, &p1);
+          fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+        }else{
+          fts3PoslistCopy(0, &p2);
+          fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+        }
+      }
+      sqlite3_free(aTmp);
+      break;
     }
   }
-  pInfo->idxNum = QUERY_GENERIC;
-  return SQLITE_OK;
-}
 
-static int fulltextDisconnect(sqlite3_vtab *pVTab){
-  FTSTRACE(("FTS3 Disconnect %p\n", pVTab));
-  fulltext_vtab_destroy((fulltext_vtab *)pVTab);
+  *pnBuffer = (int)(p-aBuffer);
   return SQLITE_OK;
 }
 
-static int fulltextDestroy(sqlite3_vtab *pVTab){
-  fulltext_vtab *v = (fulltext_vtab *)pVTab;
-  int rc;
-
-  FTSTRACE(("FTS3 Destroy %p\n", pVTab));
-  rc = sql_exec(v->db, v->zDb, v->zName,
-                "drop table if exists %_content;"
-                "drop table if exists %_segments;"
-                "drop table if exists %_segdir;"
-                );
-  if( rc!=SQLITE_OK ) return rc;
-
-  fulltext_vtab_destroy((fulltext_vtab *)pVTab);
-  return SQLITE_OK;
-}
+/* 
+** A pointer to an instance of this structure is used as the context 
+** argument to sqlite3Fts3SegReaderIterate()
+*/
+typedef struct TermSelect TermSelect;
+struct TermSelect {
+  int isReqPos;
+  char *aaOutput[16];             /* Malloc'd output buffer */
+  int anOutput[16];               /* Size of output in bytes */
+};
 
-static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
-  fulltext_cursor *c;
+/*
+** Merge all doclists in the TermSelect.aaOutput[] array into a single
+** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
+** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
+**
+** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
+** the responsibility of the caller to free any doclists left in the
+** TermSelect.aaOutput[] array.
+*/
+static int fts3TermSelectMerge(TermSelect *pTS){
+  int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
+  char *aOut = 0;
+  int nOut = 0;
+  int i;
 
-  c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor));
-  if( c ){
-    memset(c, 0, sizeof(fulltext_cursor));
-    /* sqlite will initialize c->base */
-    *ppCursor = &c->base;
-    FTSTRACE(("FTS3 Open %p: %p\n", pVTab, c));
-    return SQLITE_OK;
-  }else{
-    return SQLITE_NOMEM;
+  /* Loop through the doclists in the aaOutput[] array. Merge them all
+  ** into a single doclist.
+  */
+  for(i=0; i<SizeofArray(pTS->aaOutput); i++){
+    if( pTS->aaOutput[i] ){
+      if( !aOut ){
+        aOut = pTS->aaOutput[i];
+        nOut = pTS->anOutput[i];
+        pTS->aaOutput[0] = 0;
+      }else{
+        int nNew = nOut + pTS->anOutput[i];
+        char *aNew = sqlite3_malloc(nNew);
+        if( !aNew ){
+          sqlite3_free(aOut);
+          return SQLITE_NOMEM;
+        }
+        fts3DoclistMerge(mergetype, 0, 0,
+            aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut
+        );
+        sqlite3_free(pTS->aaOutput[i]);
+        sqlite3_free(aOut);
+        pTS->aaOutput[i] = 0;
+        aOut = aNew;
+        nOut = nNew;
+      }
+    }
   }
-}
 
-/* Free all of the dynamically allocated memory held by the
-** Snippet
-*/
-static void snippetClear(Snippet *p){
-  sqlite3_free(p->aMatch);
-  sqlite3_free(p->zOffset);
-  sqlite3_free(p->zSnippet);
-  CLEAR(p);
+  pTS->aaOutput[0] = aOut;
+  pTS->anOutput[0] = nOut;
+  return SQLITE_OK;
 }
 
 /*
-** Append a single entry to the p->aMatch[] log.
-*/
-static void snippetAppendMatch(
-  Snippet *p,               /* Append the entry to this snippet */
-  int iCol, int iTerm,      /* The column and query term */
-  int iToken,               /* Matching token in document */
-  int iStart, int nByte     /* Offset and size of the match */
+** This function is used as the sqlite3Fts3SegReaderIterate() callback when
+** querying the full-text index for a doclist associated with a term or
+** term-prefix.
+*/
+static int fts3TermSelectCb(
+  Fts3Table *p,                   /* Virtual table object */
+  void *pContext,                 /* Pointer to TermSelect structure */
+  char *zTerm,
+  int nTerm,
+  char *aDoclist,
+  int nDoclist
 ){
-  int i;
-  struct snippetMatch *pMatch;
-  if( p->nMatch+1>=p->nAlloc ){
-    p->nAlloc = p->nAlloc*2 + 10;
-    p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
-    if( p->aMatch==0 ){
-      p->nMatch = 0;
-      p->nAlloc = 0;
-      return;
-    }
-  }
-  i = p->nMatch++;
-  pMatch = &p->aMatch[i];
-  pMatch->iCol = iCol;
-  pMatch->iTerm = iTerm;
-  pMatch->iToken = iToken;
-  pMatch->iStart = iStart;
-  pMatch->nByte = nByte;
-}
+  TermSelect *pTS = (TermSelect *)pContext;
 
-/*
-** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
-*/
-#define FTS3_ROTOR_SZ   (32)
-#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1)
+  UNUSED_PARAMETER(p);
+  UNUSED_PARAMETER(zTerm);
+  UNUSED_PARAMETER(nTerm);
 
-/*
-** Function to iterate through the tokens of a compiled expression.
-**
-** Except, skip all tokens on the right-hand side of a NOT operator.
-** This function is used to find tokens as part of snippet and offset
-** generation and we do nt want snippets and offsets to report matches
-** for tokens on the RHS of a NOT.
-*/
-static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){
-  Fts3Expr *p = *ppExpr;
-  int iToken = *piToken;
-  if( iToken<0 ){
-    /* In this case the expression p is the root of an expression tree.
-    ** Move to the first token in the expression tree.
+  if( pTS->aaOutput[0]==0 ){
+    /* If this is the first term selected, copy the doclist to the output
+    ** buffer using memcpy(). TODO: Add a way to transfer control of the
+    ** aDoclist buffer from the caller so as to avoid the memcpy().
     */
-    while( p->pLeft ){
-      p = p->pLeft;
+    pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
+    pTS->anOutput[0] = nDoclist;
+    if( pTS->aaOutput[0] ){
+      memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
+    }else{
+      return SQLITE_NOMEM;
     }
-    iToken = 0;
   }else{
-    assert(p && p->eType==FTSQUERY_PHRASE );
-    if( iToken<(p->pPhrase->nToken-1) ){
-      iToken++;
-    }else{
-      iToken = 0;
-      while( p->pParent && p->pParent->pLeft!=p ){
-        assert( p->pParent->pRight==p );
-        p = p->pParent;
+    int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
+    char *aMerge = aDoclist;
+    int nMerge = nDoclist;
+    int iOut;
+
+    for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
+      char *aNew;
+      int nNew;
+      if( pTS->aaOutput[iOut]==0 ){
+        assert( iOut>0 );
+        pTS->aaOutput[iOut] = aMerge;
+        pTS->anOutput[iOut] = nMerge;
+        break;
       }
-      p = p->pParent;
-      if( p ){
-        assert( p->pRight!=0 );
-        p = p->pRight;
-        while( p->pLeft ){
-          p = p->pLeft;
+
+      nNew = nMerge + pTS->anOutput[iOut];
+      aNew = sqlite3_malloc(nNew);
+      if( !aNew ){
+        if( aMerge!=aDoclist ){
+          sqlite3_free(aMerge);
         }
+        return SQLITE_NOMEM;
+      }
+      fts3DoclistMerge(mergetype, 0, 0,
+          aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge
+      );
+
+      if( iOut>0 ) sqlite3_free(aMerge);
+      sqlite3_free(pTS->aaOutput[iOut]);
+      pTS->aaOutput[iOut] = 0;
+
+      aMerge = aNew;
+      nMerge = nNew;
+      if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
+        pTS->aaOutput[iOut] = aMerge;
+        pTS->anOutput[iOut] = nMerge;
       }
     }
   }
-
-  *ppExpr = p;
-  *piToken = iToken;
-  return p?1:0;
-}
-
-/*
-** Return TRUE if the expression node pExpr is located beneath the
-** RHS of a NOT operator.
-*/
-static int fts3ExprBeneathNot(Fts3Expr *p){
-  Fts3Expr *pParent;
-  while( p ){
-    pParent = p->pParent;
-    if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){
-      return 1;
-    }
-    p = pParent;
-  }
-  return 0;
+  return SQLITE_OK;
 }
 
 /*
-** Add entries to pSnippet->aMatch[] for every match that occurs against
-** document zDoc[0..nDoc-1] which is stored in column iColumn.
-*/
-static void snippetOffsetsOfColumn(
-  fulltext_cursor *pCur,         /* The fulltest search cursor */
-  Snippet *pSnippet,             /* The Snippet object to be filled in */
-  int iColumn,                   /* Index of fulltext table column */
-  const char *zDoc,              /* Text of the fulltext table column */
-  int nDoc                       /* Length of zDoc in bytes */
+** This function retreives the doclist for the specified term (or term
+** prefix) from the database. 
+**
+** The returned doclist may be in one of two formats, depending on the 
+** value of parameter isReqPos. If isReqPos is zero, then the doclist is
+** a sorted list of delta-compressed docids (a bare doclist). If isReqPos
+** is non-zero, then the returned list is in the same format as is stored 
+** in the database without the found length specifier at the start of on-disk
+** doclists.
+*/
+static int fts3TermSelect(
+  Fts3Table *p,                   /* Virtual table handle */
+  int iColumn,                    /* Column to query (or -ve for all columns) */
+  const char *zTerm,              /* Term to query for */
+  int nTerm,                      /* Size of zTerm in bytes */
+  int isPrefix,                   /* True for a prefix search */
+  int isReqPos,                   /* True to include position lists in output */
+  int *pnOut,                     /* OUT: Size of buffer at *ppOut */
+  char **ppOut                    /* OUT: Malloced result buffer */
 ){
-  const sqlite3_tokenizer_module *pTModule;  /* The tokenizer module */
-  sqlite3_tokenizer *pTokenizer;             /* The specific tokenizer */
-  sqlite3_tokenizer_cursor *pTCursor;        /* Tokenizer cursor */
-  fulltext_vtab *pVtab;                /* The full text index */
-  int nColumn;                         /* Number of columns in the index */
-  int i, j;                            /* Loop counters */
-  int rc;                              /* Return code */
-  unsigned int match, prevMatch;       /* Phrase search bitmasks */
-  const char *zToken;                  /* Next token from the tokenizer */
-  int nToken;                          /* Size of zToken */
-  int iBegin, iEnd, iPos;              /* Offsets of beginning and end */
-
-  /* The following variables keep a circular buffer of the last
-  ** few tokens */
-  unsigned int iRotor = 0;             /* Index of current token */
-  int iRotorBegin[FTS3_ROTOR_SZ];      /* Beginning offset of token */
-  int iRotorLen[FTS3_ROTOR_SZ];        /* Length of token */
-
-  pVtab = cursor_vtab(pCur);
-  nColumn = pVtab->nColumn;
-  pTokenizer = pVtab->pTokenizer;
-  pTModule = pTokenizer->pModule;
-  rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
-  if( rc ) return;
-  pTCursor->pTokenizer = pTokenizer;
-
-  prevMatch = 0;
-  while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){
-    Fts3Expr *pIter = pCur->pExpr;
-    int iIter = -1;
-    iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin;
-    iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin;
-    match = 0;
-    for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){
-      int nPhrase;                    /* Number of tokens in current phrase */
-      struct PhraseToken *pToken;     /* Current token */
-      int iCol;                       /* Column index */
-
-      if( fts3ExprBeneathNot(pIter) ) continue;
-      nPhrase = pIter->pPhrase->nToken;
-      pToken = &pIter->pPhrase->aToken[iIter];
-      iCol = pIter->pPhrase->iColumn;
-      if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
-      if( pToken->n>nToken ) continue;
-      if( !pToken->isPrefix && pToken->n<nToken ) continue;
-      assert( pToken->n<=nToken );
-      if( memcmp(pToken->z, zToken, pToken->n) ) continue;
-      if( iIter>0 && (prevMatch & (1<<i))==0 ) continue;
-      match |= 1<<i;
-      if( i==(FTS3_ROTOR_SZ-2) || nPhrase==iIter+1 ){
-        for(j=nPhrase-1; j>=0; j--){
-          int k = (iRotor-j) & FTS3_ROTOR_MASK;
-          snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
-                iRotorBegin[k], iRotorLen[k]);
-        }
+  int i;
+  TermSelect tsc;
+  Fts3SegFilter filter;           /* Segment term filter configuration */
+  Fts3SegReader **apSegment;      /* Array of segments to read data from */
+  int nSegment = 0;               /* Size of apSegment array */
+  int nAlloc = 16;                /* Allocated size of segment array */
+  int rc;                         /* Return code */
+  sqlite3_stmt *pStmt = 0;        /* SQL statement to scan %_segdir table */
+  int iAge = 0;                   /* Used to assign ages to segments */
+
+  apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader*)*nAlloc);
+  if( !apSegment ) return SQLITE_NOMEM;
+  rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &apSegment[0]);
+  if( rc!=SQLITE_OK ) goto finished;
+  if( apSegment[0] ){
+    nSegment = 1;
+  }
+
+  /* Loop through the entire %_segdir table. For each segment, create a
+  ** Fts3SegReader to iterate through the subset of the segment leaves
+  ** that may contain a term that matches zTerm/nTerm. For non-prefix
+  ** searches, this is always a single leaf. For prefix searches, this
+  ** may be a contiguous block of leaves.
+  **
+  ** The code in this loop does not actually load any leaves into memory
+  ** (unless the root node happens to be a leaf). It simply examines the
+  ** b-tree structure to determine which leaves need to be inspected.
+  */
+  rc = sqlite3Fts3AllSegdirs(p, &pStmt);
+  while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+    Fts3SegReader *pNew = 0;
+    int nRoot = sqlite3_column_bytes(pStmt, 4);
+    char const *zRoot = sqlite3_column_blob(pStmt, 4);
+    if( sqlite3_column_int64(pStmt, 1)==0 ){
+      /* The entire segment is stored on the root node (which must be a
+      ** leaf). Do not bother inspecting any data in this case, just
+      ** create a Fts3SegReader to scan the single leaf. 
+      */
+      rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
+    }else{
+      int rc2;                    /* Return value of sqlite3Fts3ReadBlock() */
+      sqlite3_int64 i1;           /* Blockid of leaf that may contain zTerm */
+      rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
+      if( rc==SQLITE_OK ){
+        sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
+        rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
       }
-    }
-    prevMatch = match<<1;
-    iRotor++;
-  }
-  pTModule->xClose(pTCursor);  
-}
 
-/*
-** Remove entries from the pSnippet structure to account for the NEAR
-** operator. When this is called, pSnippet contains the list of token 
-** offsets produced by treating all NEAR operators as AND operators.
-** This function removes any entries that should not be present after
-** accounting for the NEAR restriction. For example, if the queried
-** document is:
-**
-**     "A B C D E A"
-**
-** and the query is:
-** 
-**     A NEAR/0 E
-**
-** then when this function is called the Snippet contains token offsets
-** 0, 4 and 5. This function removes the "0" entry (because the first A
-** is not near enough to an E).
-**
-** When this function is called, the value pointed to by parameter piLeft is
-** the integer id of the left-most token in the expression tree headed by
-** pExpr. This function increments *piLeft by the total number of tokens
-** in the expression tree headed by pExpr.
-**
-** Return 1 if any trimming occurs.  Return 0 if no trimming is required.
-*/
-static int trimSnippetOffsets(
-  Fts3Expr *pExpr,      /* The search expression */
-  Snippet *pSnippet,    /* The set of snippet offsets to be trimmed */
-  int *piLeft           /* Index of left-most token in pExpr */
-){
-  if( pExpr ){
-    if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){
-      return 1;
+      /* The following call to ReadBlock() serves to reset the SQL statement
+      ** used to retrieve blocks of data from the %_segments table. If it is
+      ** not reset here, then it may remain classified as an active statement 
+      ** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands 
+      ** failing.
+      */ 
+      rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0);
+      if( rc==SQLITE_OK ){
+        rc = rc2;
+      }
     }
+    iAge++;
 
-    switch( pExpr->eType ){
-      case FTSQUERY_PHRASE:
-        *piLeft += pExpr->pPhrase->nToken;
-        break;
-      case FTSQUERY_NEAR: {
-        /* The right-hand-side of a NEAR operator is always a phrase. The
-        ** left-hand-side is either a phrase or an expression tree that is 
-        ** itself headed by a NEAR operator. The following initializations
-        ** set local variable iLeft to the token number of the left-most
-        ** token in the right-hand phrase, and iRight to the right most
-        ** token in the same phrase. For example, if we had:
-        **
-        **     <col> MATCH '"abc def" NEAR/2 "ghi jkl"'
-        **
-        ** then iLeft will be set to 2 (token number of ghi) and nToken will
-        ** be set to 4.
-        */
-        Fts3Expr *pLeft = pExpr->pLeft;
-        Fts3Expr *pRight = pExpr->pRight;
-        int iLeft = *piLeft;
-        int nNear = pExpr->nNear;
-        int nToken = pRight->pPhrase->nToken;
-        int jj, ii;
-        if( pLeft->eType==FTSQUERY_NEAR ){
-          pLeft = pLeft->pRight;
-        }
-        assert( pRight->eType==FTSQUERY_PHRASE );
-        assert( pLeft->eType==FTSQUERY_PHRASE );
-        nToken += pLeft->pPhrase->nToken;
-
-        for(ii=0; ii<pSnippet->nMatch; ii++){
-          struct snippetMatch *p = &pSnippet->aMatch[ii];
-          if( p->iTerm==iLeft ){
-            int isOk = 0;
-            /* Snippet ii is an occurence of query term iLeft in the document.
-            ** It occurs at position (p->iToken) of the document. We now
-            ** search for an instance of token (iLeft-1) somewhere in the 
-            ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within 
-            ** the set of snippetMatch structures. If one is found, proceed. 
-            ** If one cannot be found, then remove snippets ii..(ii+N-1) 
-            ** from the matching snippets, where N is the number of tokens 
-            ** in phrase pRight->pPhrase.
-            */
-            for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
-              struct snippetMatch *p2 = &pSnippet->aMatch[jj];
-              if( p2->iTerm==(iLeft-1) ){
-                if( p2->iToken>=(p->iToken-nNear-1) 
-                 && p2->iToken<(p->iToken+nNear+nToken) 
-                ){
-                  isOk = 1;
-                }
-              }
-            }
-            if( !isOk ){
-              int kk;
-              for(kk=0; kk<pRight->pPhrase->nToken; kk++){
-                pSnippet->aMatch[kk+ii].iTerm = -2;
-              }
-              return 1;
-            }
-          }
-          if( p->iTerm==(iLeft-1) ){
-            int isOk = 0;
-            for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
-              struct snippetMatch *p2 = &pSnippet->aMatch[jj];
-              if( p2->iTerm==iLeft ){
-                if( p2->iToken<=(p->iToken+nNear+1) 
-                 && p2->iToken>(p->iToken-nNear-nToken) 
-                ){
-                  isOk = 1;
-                }
-              }
-            }
-            if( !isOk ){
-              int kk;
-              for(kk=0; kk<pLeft->pPhrase->nToken; kk++){
-                pSnippet->aMatch[ii-kk].iTerm = -2;
-              }
-              return 1;
-            }
-          }
+    /* If a new Fts3SegReader was allocated, add it to the apSegment array. */
+    assert( pNew!=0 || rc!=SQLITE_OK );
+    if( pNew ){
+      if( nSegment==nAlloc ){
+        Fts3SegReader **pArray;
+        nAlloc += 16;
+        pArray = (Fts3SegReader **)sqlite3_realloc(
+            apSegment, nAlloc*sizeof(Fts3SegReader *)
+        );
+        if( !pArray ){
+          sqlite3Fts3SegReaderFree(p, pNew);
+          rc = SQLITE_NOMEM;
+          goto finished;
         }
-        break;
+        apSegment = pArray;
       }
+      apSegment[nSegment++] = pNew;
     }
-
-    if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){
-      return 1;
-    }
-  }
-  return 0;
-}
-
-/*
-** Compute all offsets for the current row of the query.  
-** If the offsets have already been computed, this routine is a no-op.
-*/
-static void snippetAllOffsets(fulltext_cursor *p){
-  int nColumn;
-  int iColumn, i;
-  int iFirst, iLast;
-  int iTerm = 0;
-  fulltext_vtab *pFts = cursor_vtab(p);
-
-  if( p->snippet.nMatch || p->pExpr==0 ){
-    return;
-  }
-  nColumn = pFts->nColumn;
-  iColumn = (p->iCursorType - QUERY_FULLTEXT);
-  if( iColumn<0 || iColumn>=nColumn ){
-    /* Look for matches over all columns of the full-text index */
-    iFirst = 0;
-    iLast = nColumn-1;
-  }else{
-    /* Look for matches in the iColumn-th column of the index only */
-    iFirst = iColumn;
-    iLast = iColumn;
   }
-  for(i=iFirst; i<=iLast; i++){
-    const char *zDoc;
-    int nDoc;
-    zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
-    nDoc = sqlite3_column_bytes(p->pStmt, i+1);
-    snippetOffsetsOfColumn(p, &p->snippet, i, zDoc, nDoc);
+  if( rc!=SQLITE_DONE ){
+    assert( rc!=SQLITE_OK );
+    goto finished;
   }
 
-  while( trimSnippetOffsets(p->pExpr, &p->snippet, &iTerm) ){
-    iTerm = 0;
-  }
-}
+  memset(&tsc, 0, sizeof(TermSelect));
+  tsc.isReqPos = isReqPos;
 
-/*
-** Convert the information in the aMatch[] array of the snippet
-** into the string zOffset[0..nOffset-1]. This string is used as
-** the return of the SQL offsets() function.
-*/
-static void snippetOffsetText(Snippet *p){
-  int i;
-  int cnt = 0;
-  StringBuffer sb;
-  char zBuf[200];
-  if( p->zOffset ) return;
-  initStringBuffer(&sb);
-  for(i=0; i<p->nMatch; i++){
-    struct snippetMatch *pMatch = &p->aMatch[i];
-    if( pMatch->iTerm>=0 ){
-      /* If snippetMatch.iTerm is less than 0, then the match was 
-      ** discarded as part of processing the NEAR operator (see the 
-      ** trimSnippetOffsetsForNear() function for details). Ignore 
-      ** it in this case
-      */
-      zBuf[0] = ' ';
-      sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
-          pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
-      append(&sb, zBuf);
-      cnt++;
-    }
-  }
-  p->zOffset = stringBufferData(&sb);
-  p->nOffset = stringBufferLength(&sb);
-}
+  filter.flags = FTS3_SEGMENT_IGNORE_EMPTY 
+        | (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
+        | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
+        | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
+  filter.iCol = iColumn;
+  filter.zTerm = zTerm;
+  filter.nTerm = nTerm;
 
-/*
-** zDoc[0..nDoc-1] is phrase of text.  aMatch[0..nMatch-1] are a set
-** of matching words some of which might be in zDoc.  zDoc is column
-** number iCol.
-**
-** iBreak is suggested spot in zDoc where we could begin or end an
-** excerpt.  Return a value similar to iBreak but possibly adjusted
-** to be a little left or right so that the break point is better.
-*/
-static int wordBoundary(
-  int iBreak,                   /* The suggested break point */
-  const char *zDoc,             /* Document text */
-  int nDoc,                     /* Number of bytes in zDoc[] */
-  struct snippetMatch *aMatch,  /* Matching words */
-  int nMatch,                   /* Number of entries in aMatch[] */
-  int iCol                      /* The column number for zDoc[] */
-){
-  int i;
-  if( iBreak<=10 ){
-    return 0;
-  }
-  if( iBreak>=nDoc-10 ){
-    return nDoc;
+  rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
+      fts3TermSelectCb, (void *)&tsc
+  );
+  if( rc==SQLITE_OK ){
+    rc = fts3TermSelectMerge(&tsc);
   }
-  for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
-  while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
-  if( i<nMatch ){
-    if( aMatch[i].iStart<iBreak+10 ){
-      return aMatch[i].iStart;
-    }
-    if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
-      return aMatch[i-1].iStart;
+
+  if( rc==SQLITE_OK ){
+    *ppOut = tsc.aaOutput[0];
+    *pnOut = tsc.anOutput[0];
+  }else{
+    for(i=0; i<SizeofArray(tsc.aaOutput); i++){
+      sqlite3_free(tsc.aaOutput[i]);
     }
   }
-  for(i=1; i<=10; i++){
-    if( safe_isspace(zDoc[iBreak-i]) ){
-      return iBreak - i + 1;
-    }
-    if( safe_isspace(zDoc[iBreak+i]) ){
-      return iBreak + i + 1;
-    }
+
+finished:
+  sqlite3_reset(pStmt);
+  for(i=0; i<nSegment; i++){
+    sqlite3Fts3SegReaderFree(p, apSegment[i]);
   }
-  return iBreak;
+  sqlite3_free(apSegment);
+  return rc;
 }
 
 
-
-/*
-** Allowed values for Snippet.aMatch[].snStatus
-*/
-#define SNIPPET_IGNORE  0   /* It is ok to omit this match from the snippet */
-#define SNIPPET_DESIRED 1   /* We want to include this match in the snippet */
-
-/*
-** Generate the text of a snippet.
+/* 
+** Return a DocList corresponding to the phrase *pPhrase.
 */
-static void snippetText(
-  fulltext_cursor *pCursor,   /* The cursor we need the snippet for */
-  const char *zStartMark,     /* Markup to appear before each match */
-  const char *zEndMark,       /* Markup to appear after each match */
-  const char *zEllipsis       /* Ellipsis mark */
+static int fts3PhraseSelect(
+  Fts3Table *p,                   /* Virtual table handle */
+  Fts3Phrase *pPhrase,            /* Phrase to return a doclist for */
+  int isReqPos,                   /* True if output should contain positions */
+  char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
+  int *pnOut                      /* OUT: Size of buffer at *paOut */
 ){
-  int i, j;
-  struct snippetMatch *aMatch;
-  int nMatch;
-  int nDesired;
-  StringBuffer sb;
-  int tailCol;
-  int tailOffset;
-  int iCol;
-  int nDoc;
-  const char *zDoc;
-  int iStart, iEnd;
-  int tailEllipsis = 0;
-  int iMatch;
-  
+  char *pOut = 0;
+  int nOut = 0;
+  int rc = SQLITE_OK;
+  int ii;
+  int iCol = pPhrase->iColumn;
+  int isTermPos = (pPhrase->nToken>1 || isReqPos);
 
-  sqlite3_free(pCursor->snippet.zSnippet);
-  pCursor->snippet.zSnippet = 0;
-  aMatch = pCursor->snippet.aMatch;
-  nMatch = pCursor->snippet.nMatch;
-  initStringBuffer(&sb);
+  for(ii=0; ii<pPhrase->nToken; ii++){
+    struct PhraseToken *pTok = &pPhrase->aToken[ii];
+    char *z = pTok->z;            /* Next token of the phrase */
+    int n = pTok->n;              /* Size of z in bytes */
+    int isPrefix = pTok->isPrefix;/* True if token is a prefix */
+    char *pList;                  /* Pointer to token doclist */
+    int nList;                    /* Size of buffer at pList */
 
-  for(i=0; i<nMatch; i++){
-    aMatch[i].snStatus = SNIPPET_IGNORE;
-  }
-  nDesired = 0;
-  for(i=0; i<FTS3_ROTOR_SZ; i++){
-    for(j=0; j<nMatch; j++){
-      if( aMatch[j].iTerm==i ){
-        aMatch[j].snStatus = SNIPPET_DESIRED;
-        nDesired++;
-        break;
-      }
-    }
-  }
+    rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList);
+    if( rc!=SQLITE_OK ) break;
 
-  iMatch = 0;
-  tailCol = -1;
-  tailOffset = 0;
-  for(i=0; i<nMatch && nDesired>0; i++){
-    if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
-    nDesired--;
-    iCol = aMatch[i].iCol;
-    zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
-    nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
-    iStart = aMatch[i].iStart - 40;
-    iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
-    if( iStart<=10 ){
-      iStart = 0;
-    }
-    if( iCol==tailCol && iStart<=tailOffset+20 ){
-      iStart = tailOffset;
-    }
-    if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){
-      trimWhiteSpace(&sb);
-      appendWhiteSpace(&sb);
-      append(&sb, zEllipsis);
-      appendWhiteSpace(&sb);
-    }
-    iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
-    iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
-    if( iEnd>=nDoc-10 ){
-      iEnd = nDoc;
-      tailEllipsis = 0;
+    if( ii==0 ){
+      pOut = pList;
+      nOut = nList;
     }else{
-      tailEllipsis = 1;
-    }
-    while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
-    while( iStart<iEnd ){
-      while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
-             && aMatch[iMatch].iCol<=iCol ){
-        iMatch++;
-      }
-      if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
-             && aMatch[iMatch].iCol==iCol ){
-        nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
-        iStart = aMatch[iMatch].iStart;
-        append(&sb, zStartMark);
-        nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
-        append(&sb, zEndMark);
-        iStart += aMatch[iMatch].nByte;
-        for(j=iMatch+1; j<nMatch; j++){
-          if( aMatch[j].iTerm==aMatch[iMatch].iTerm
-              && aMatch[j].snStatus==SNIPPET_DESIRED ){
-            nDesired--;
-            aMatch[j].snStatus = SNIPPET_IGNORE;
-          }
-        }
-      }else{
-        nappend(&sb, &zDoc[iStart], iEnd - iStart);
-        iStart = iEnd;
+      /* Merge the new term list and the current output. If this is the
+      ** last term in the phrase, and positions are not required in the
+      ** output of this function, the positions can be dropped as part
+      ** of this merge. Either way, the result of this merge will be
+      ** smaller than nList bytes. The code in fts3DoclistMerge() is written
+      ** so that it is safe to use pList as the output as well as an input
+      ** in this case.
+      */
+      int mergetype = MERGE_POS_PHRASE;
+      if( ii==pPhrase->nToken-1 && !isReqPos ){
+        mergetype = MERGE_PHRASE;
       }
+      fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList);
+      sqlite3_free(pOut);
+      pOut = pList;
     }
-    tailCol = iCol;
-    tailOffset = iEnd;
-  }
-  trimWhiteSpace(&sb);
-  if( tailEllipsis ){
-    appendWhiteSpace(&sb);
-    append(&sb, zEllipsis);
+    assert( nOut==0 || pOut!=0 );
   }
-  pCursor->snippet.zSnippet = stringBufferData(&sb);
-  pCursor->snippet.nSnippet = stringBufferLength(&sb);
-}
 
-
-/*
-** Close the cursor.  For additional information see the documentation
-** on the xClose method of the virtual table interface.
-*/
-static int fulltextClose(sqlite3_vtab_cursor *pCursor){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-  FTSTRACE(("FTS3 Close %p\n", c));
-  sqlite3_finalize(c->pStmt);
-  sqlite3Fts3ExprFree(c->pExpr);
-  snippetClear(&c->snippet);
-  if( c->result.nData!=0 ){
-    dlrDestroy(&c->reader);
+  if( rc==SQLITE_OK ){
+    *paOut = pOut;
+    *pnOut = nOut;
+  }else{
+    sqlite3_free(pOut);
   }
-  dataBufferDestroy(&c->result);
-  sqlite3_free(c);
-  return SQLITE_OK;
+  return rc;
 }
 
-static int fulltextNext(sqlite3_vtab_cursor *pCursor){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
+static int fts3NearMerge(
+  int mergetype,                  /* MERGE_POS_NEAR or MERGE_NEAR */
+  int nNear,                      /* Parameter to NEAR operator */
+  int nTokenLeft,                 /* Number of tokens in LHS phrase arg */
+  char *aLeft,                    /* Doclist for LHS (incl. positions) */
+  int nLeft,                      /* Size of LHS doclist in bytes */
+  int nTokenRight,                /* As nTokenLeft */
+  char *aRight,                   /* As aLeft */
+  int nRight,                     /* As nRight */
+  char **paOut,                   /* OUT: Results of merge (malloced) */
+  int *pnOut                      /* OUT: Sized of output buffer */
+){
+  char *aOut;
   int rc;
 
-  FTSTRACE(("FTS3 Next %p\n", pCursor));
-  snippetClear(&c->snippet);
-  if( c->iCursorType < QUERY_FULLTEXT ){
-    /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
-    rc = sqlite3_step(c->pStmt);
-    switch( rc ){
-      case SQLITE_ROW:
-        c->eof = 0;
-        return SQLITE_OK;
-      case SQLITE_DONE:
-        c->eof = 1;
-        return SQLITE_OK;
-      default:
-        c->eof = 1;
-        return rc;
-    }
-  } else {  /* full-text query */
-    rc = sqlite3_reset(c->pStmt);
-    if( rc!=SQLITE_OK ) return rc;
+  assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
 
-    if( c->result.nData==0 || dlrAtEnd(&c->reader) ){
-      c->eof = 1;
-      return SQLITE_OK;
-    }
-    rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
-    if( rc!=SQLITE_OK ) return rc;
-    rc = dlrStep(&c->reader);
-    if( rc!=SQLITE_OK ) return rc;
-    /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
-    rc = sqlite3_step(c->pStmt);
-    if( rc==SQLITE_ROW ){   /* the case we expect */
-      c->eof = 0;
-      return SQLITE_OK;
+  aOut = sqlite3_malloc(nLeft+nRight+1);
+  if( aOut==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, 
+      aOut, pnOut, aLeft, nLeft, aRight, nRight
+    );
+    if( rc!=SQLITE_OK ){
+      sqlite3_free(aOut);
+      aOut = 0;
     }
-    /* Corrupt if the index refers to missing document. */
-    if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
-
-    return rc;
   }
-}
-
-
-/* TODO(shess) If we pushed LeafReader to the top of the file, or to
-** another file, term_select() could be pushed above
-** docListOfTerm().
-*/
-static int termSelect(fulltext_vtab *v, int iColumn,
-                      const char *pTerm, int nTerm, int isPrefix,
-                      DocListType iType, DataBuffer *out);
 
-/* 
-** Return a DocList corresponding to the phrase *pPhrase.
-**
-** The resulting DL_DOCIDS doclist is stored in pResult, which is
-** overwritten.
-*/
-static int docListOfPhrase(
-  fulltext_vtab *pTab,   /* The full text index */
-  Fts3Phrase *pPhrase,   /* Phrase to return a doclist corresponding to */
-  DocListType eListType, /* Either DL_DOCIDS or DL_POSITIONS */
-  DataBuffer *pResult    /* Write the result here */
-){
-  int ii;
-  int rc = SQLITE_OK;
-  int iCol = pPhrase->iColumn;
-  DocListType eType = eListType;
-  assert( eType==DL_POSITIONS || eType==DL_DOCIDS );
-  if( pPhrase->nToken>1 ){
-    eType = DL_POSITIONS;
-  }
+  *paOut = aOut;
+  return rc;
+}
 
-  /* This code should never be called with buffered updates. */
-  assert( pTab->nPendingData<0 );
+int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
+  int rc;
+  if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
+    sqlite3_free(pLeft->aDoclist);
+    sqlite3_free(pRight->aDoclist);
+    pRight->aDoclist = 0;
+    pLeft->aDoclist = 0;
+    rc = SQLITE_OK;
+  }else{
+    char *aOut;
+    int nOut;
 
-  for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
-    DataBuffer tmp;
-    struct PhraseToken *p = &pPhrase->aToken[ii];
-    rc = termSelect(pTab, iCol, p->z, p->n, p->isPrefix, eType, &tmp);
-    if( rc==SQLITE_OK ){
-      if( ii==0 ){
-        *pResult = tmp;
-      }else{
-        DataBuffer res = *pResult;
-        dataBufferInit(pResult, 0);
-        if( ii==(pPhrase->nToken-1) ){
-          eType = eListType;
-        }
-        rc = docListPhraseMerge(
-          res.pData, res.nData, tmp.pData, tmp.nData, 0, 0, eType, pResult
-        );
-        dataBufferDestroy(&res);
-        dataBufferDestroy(&tmp);
-        if( rc!= SQLITE_OK ) return rc;
-      }
-    }
+    rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
+        pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
+        pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
+        &aOut, &nOut
+    );
+    if( rc!=SQLITE_OK ) return rc;
+    sqlite3_free(pRight->aDoclist);
+    pRight->aDoclist = aOut;
+    pRight->nDoclist = nOut;
+
+    rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
+        pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
+        pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
+        &aOut, &nOut
+    );
+    sqlite3_free(pLeft->aDoclist);
+    pLeft->aDoclist = aOut;
+    pLeft->nDoclist = nOut;
   }
-
   return rc;
 }
 
 /*
-** Evaluate the full-text expression pExpr against fts3 table pTab. Write
-** the results into pRes.
+** Evaluate the full-text expression pExpr against fts3 table pTab. Store
+** the resulting doclist in *paOut and *pnOut.  This routine mallocs for
+** the space needed to store the output.  The caller is responsible for
+** freeing the space when it has finished.
 */
 static int evalFts3Expr(
-  fulltext_vtab *pTab,           /* Fts3 Virtual table object */
-  Fts3Expr *pExpr,               /* Parsed fts3 expression */
-  DataBuffer *pRes               /* OUT: Write results of the expression here */
+  Fts3Table *p,                   /* Virtual table handle */
+  Fts3Expr *pExpr,                /* Parsed fts3 expression */
+  char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
+  int *pnOut,                     /* OUT: Size of buffer at *paOut */
+  int isReqPos                    /* Require positions in output buffer */
 ){
-  int rc = SQLITE_OK;
+  int rc = SQLITE_OK;             /* Return code */
 
-  /* Initialize the output buffer. If this is an empty query (pExpr==0), 
-  ** this is all that needs to be done. Empty queries produce empty 
-  ** result sets.
-  */
-  dataBufferInit(pRes, 0);
+  /* Zero the output parameters. */
+  *paOut = 0;
+  *pnOut = 0;
 
   if( pExpr ){
+    assert( pExpr->eType==FTSQUERY_PHRASE 
+         || pExpr->eType==FTSQUERY_NEAR 
+         || isReqPos==0
+    );
     if( pExpr->eType==FTSQUERY_PHRASE ){
-      DocListType eType = DL_DOCIDS;
-      if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
-        eType = DL_POSITIONS;
-      }
-      rc = docListOfPhrase(pTab, pExpr->pPhrase, eType, pRes);
+      rc = fts3PhraseSelect(p, pExpr->pPhrase, 
+          isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
+          paOut, pnOut
+      );
     }else{
-      DataBuffer lhs;
-      DataBuffer rhs;
+      char *aLeft;
+      char *aRight;
+      int nLeft;
+      int nRight;
 
-      dataBufferInit(&rhs, 0);
-      if( SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pLeft, &lhs)) 
-       && SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pRight, &rhs)) 
+      if( 0==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
+       && 0==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
       ){
+        assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR     
+            || pExpr->eType==FTSQUERY_AND  || pExpr->eType==FTSQUERY_NOT
+        );
         switch( pExpr->eType ){
           case FTSQUERY_NEAR: {
-            int nToken;
             Fts3Expr *pLeft;
-            DocListType eType = DL_DOCIDS;
+            Fts3Expr *pRight;
+            int mergetype = isReqPos ? MERGE_POS_NEAR : MERGE_NEAR;
+           
             if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
-              eType = DL_POSITIONS;
+              mergetype = MERGE_POS_NEAR;
             }
             pLeft = pExpr->pLeft;
             while( pLeft->eType==FTSQUERY_NEAR ){ 
               pLeft=pLeft->pRight;
             }
-            assert( pExpr->pRight->eType==FTSQUERY_PHRASE );
+            pRight = pExpr->pRight;
+            assert( pRight->eType==FTSQUERY_PHRASE );
             assert( pLeft->eType==FTSQUERY_PHRASE );
-            nToken = pLeft->pPhrase->nToken + pExpr->pRight->pPhrase->nToken;
-            rc = docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, 
-                pExpr->nNear+1, nToken, eType, pRes
+
+            rc = fts3NearMerge(mergetype, pExpr->nNear, 
+                pLeft->pPhrase->nToken, aLeft, nLeft,
+                pRight->pPhrase->nToken, aRight, nRight,
+                paOut, pnOut
             );
+            sqlite3_free(aLeft);
             break;
           }
-          case FTSQUERY_NOT: {
-            rc = docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes);
-            break;
-          }
-          case FTSQUERY_AND: {
-            rc = docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes);
+
+          case FTSQUERY_OR: {
+            /* Allocate a buffer for the output. The maximum size is the
+            ** sum of the sizes of the two input buffers. The +1 term is
+            ** so that a buffer of zero bytes is never allocated - this can
+            ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
+            */
+            char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
+            rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
+                aLeft, nLeft, aRight, nRight
+            );
+            *paOut = aBuffer;
+            sqlite3_free(aLeft);
             break;
           }
-          case FTSQUERY_OR: {
-            rc = docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes);
+
+          default: {
+            assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
+            fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
+                aLeft, nLeft, aRight, nRight
+            );
+            *paOut = aLeft;
             break;
           }
         }
       }
-      dataBufferDestroy(&lhs);
-      dataBufferDestroy(&rhs);
+      sqlite3_free(aRight);
     }
   }
 
   return rc;
 }
 
-/* TODO(shess) Refactor the code to remove this forward decl. */
-static int flushPendingTerms(fulltext_vtab *v);
-
-/* Perform a full-text query using the search expression in
-** zInput[0..nInput-1].  Return a list of matching documents
-** in pResult.
-**
-** Queries must match column iColumn.  Or if iColumn>=nColumn
-** they are allowed to match against any column.
-*/
-static int fulltextQuery(
-  fulltext_vtab *v,      /* The full text index */
-  int iColumn,           /* Match against this column by default */
-  const char *zInput,    /* The query string */
-  int nInput,            /* Number of bytes in zInput[] */
-  DataBuffer *pResult,   /* Write the result doclist here */
-  Fts3Expr **ppExpr        /* Put parsed query string here */
-){
-  int rc;
-
-  /* TODO(shess) Instead of flushing pendingTerms, we could query for
-  ** the relevant term and merge the doclist into what we receive from
-  ** the database.  Wait and see if this is a common issue, first.
-  **
-  ** A good reason not to flush is to not generate update-related
-  ** error codes from here.
-  */
-
-  /* Flush any buffered updates before executing the query. */
-  rc = flushPendingTerms(v);
-  if( rc!=SQLITE_OK ){
-    return rc;
-  }
-
-  /* Parse the query passed to the MATCH operator. */
-  rc = sqlite3Fts3ExprParse(v->pTokenizer, 
-      v->azColumn, v->nColumn, iColumn, zInput, nInput, ppExpr
-  );
-  if( rc!=SQLITE_OK ){
-    assert( 0==(*ppExpr) );
-    return rc;
-  }
-
-  return evalFts3Expr(v, *ppExpr, pResult);
-}
-
 /*
 ** This is the xFilter interface for the virtual table.  See
 ** the virtual table xFilter method documentation for additional
 ** information.
 **
-** If idxNum==QUERY_GENERIC then do a full table scan against
+** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against
 ** the %_content table.
 **
-** If idxNum==QUERY_DOCID then do a docid lookup for a single entry
+** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry
 ** in the %_content table.
 **
-** If idxNum>=QUERY_FULLTEXT then use the full text index.  The
+** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index.  The
 ** column on the left-hand side of the MATCH operator is column
-** number idxNum-QUERY_FULLTEXT, 0 indexed.  argv[0] is the right-hand
+** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed.  argv[0] is the right-hand
 ** side of the MATCH operator.
 */
 /* TODO(shess) Upgrade the cursor initialization and destruction to
-** account for fulltextFilter() being called multiple times on the
-** same cursor.  The current solution is very fragile.  Apply fix to
+** account for fts3FilterMethod() being called multiple times on the
+** same cursor. The current solution is very fragile. Apply fix to
 ** fts3 as appropriate.
 */
-static int fulltextFilter(
-  sqlite3_vtab_cursor *pCursor,     /* The cursor used for this query */
-  int idxNum, const char *idxStr,   /* Which indexing scheme to use */
-  int argc, sqlite3_value **argv    /* Arguments for the indexing scheme */
+static int fts3FilterMethod(
+  sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
+  int idxNum,                     /* Strategy index */
+  const char *idxStr,             /* Unused */
+  int nVal,                       /* Number of elements in apVal */
+  sqlite3_value **apVal           /* Arguments for the indexing scheme */
 ){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-  fulltext_vtab *v = cursor_vtab(c);
-  int rc;
-
-  FTSTRACE(("FTS3 Filter %p\n",pCursor));
-
-  /* If the cursor has a statement that was not prepared according to
-  ** idxNum, clear it.  I believe all calls to fulltextFilter with a
-  ** given cursor will have the same idxNum , but in this case it's
-  ** easy to be safe.
+  const char *azSql[] = {
+    "SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */
+    "SELECT * FROM %Q.'%q_content'",                 /* full-table-scan */
+  };
+  int rc;                         /* Return code */
+  char *zSql;                     /* SQL statement used to access %_content */
+  Fts3Table *p = (Fts3Table *)pCursor->pVtab;
+  Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+
+  UNUSED_PARAMETER(idxStr);
+  UNUSED_PARAMETER(nVal);
+
+  assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
+  assert( nVal==0 || nVal==1 );
+  assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
+
+  /* In case the cursor has been used before, clear it now. */
+  sqlite3_finalize(pCsr->pStmt);
+  sqlite3_free(pCsr->aDoclist);
+  sqlite3Fts3ExprFree(pCsr->pExpr);
+  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
+
+  /* Compile a SELECT statement for this cursor. For a full-table-scan, the
+  ** statement loops through all rows of the %_content table. For a
+  ** full-text query or docid lookup, the statement retrieves a single
+  ** row by docid.
   */
-  if( c->pStmt && c->iCursorType!=idxNum ){
-    sqlite3_finalize(c->pStmt);
-    c->pStmt = NULL;
-  }
-
-  /* Get a fresh statement appropriate to idxNum. */
-  /* TODO(shess): Add a prepared-statement cache in the vt structure.
-  ** The cache must handle multiple open cursors.  Easier to cache the
-  ** statement variants at the vt to reduce malloc/realloc/free here.
-  ** Or we could have a StringBuffer variant which allowed stack
-  ** construction for small values.
-  */
-  if( !c->pStmt ){
-    StringBuffer sb;
-    initStringBuffer(&sb);
-    append(&sb, "SELECT docid, ");
-    appendList(&sb, v->nColumn, v->azContentColumn);
-    append(&sb, " FROM %_content");
-    if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?");
-    rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt,
-                     stringBufferData(&sb));
-    stringBufferDestroy(&sb);
-    if( rc!=SQLITE_OK ) return rc;
-    c->iCursorType = idxNum;
+  zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
+  if( !zSql ){
+    rc = SQLITE_NOMEM;
   }else{
-    sqlite3_reset(c->pStmt);
-    assert( c->iCursorType==idxNum );
-  }
-
-  switch( idxNum ){
-    case QUERY_GENERIC:
-      break;
-
-    case QUERY_DOCID:
-      rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
-      if( rc!=SQLITE_OK ) return rc;
-      break;
-
-    default:   /* full-text search */
-    {
-      int iCol = idxNum-QUERY_FULLTEXT;
-      const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
-      assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
-      assert( argc==1 );
-      if( c->result.nData!=0 ){
-        /* This case happens if the same cursor is used repeatedly. */
-        dlrDestroy(&c->reader);
-        dataBufferReset(&c->result);
-      }else{
-        dataBufferInit(&c->result, 0);
-      }
-      rc = fulltextQuery(v, iCol, zQuery, -1, &c->result, &c->pExpr);
-      if( rc!=SQLITE_OK ) return rc;
-      if( c->result.nData!=0 ){
-        dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
-      }
-      break;
-    }
-  }
-
-  return fulltextNext(pCursor);
-}
-
-/* This is the xEof method of the virtual table.  The SQLite core
-** calls this routine to find out if it has reached the end of
-** a query's results set.
-*/
-static int fulltextEof(sqlite3_vtab_cursor *pCursor){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-  return c->eof;
-}
-
-/* This is the xColumn method of the virtual table.  The SQLite
-** core calls this method during a query when it needs the value
-** of a column from the virtual table.  This method needs to use
-** one of the sqlite3_result_*() routines to store the requested
-** value back in the pContext.
-*/
-static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
-                          sqlite3_context *pContext, int idxCol){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-  fulltext_vtab *v = cursor_vtab(c);
-
-  if( idxCol<v->nColumn ){
-    sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
-    sqlite3_result_value(pContext, pVal);
-  }else if( idxCol==v->nColumn ){
-    /* The extra column whose name is the same as the table.
-    ** Return a blob which is a pointer to the cursor
-    */
-    sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT);
-  }else if( idxCol==v->nColumn+1 ){
-    /* The docid column, which is an alias for rowid. */
-    sqlite3_value *pVal = sqlite3_column_value(c->pStmt, 0);
-    sqlite3_result_value(pContext, pVal);
-  }
-  return SQLITE_OK;
-}
-
-/* This is the xRowid method.  The SQLite core calls this routine to
-** retrieve the rowid for the current row of the result set.  fts3
-** exposes %_content.docid as the rowid for the virtual table.  The
-** rowid should be written to *pRowid.
-*/
-static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
-  fulltext_cursor *c = (fulltext_cursor *) pCursor;
-
-  *pRowid = sqlite3_column_int64(c->pStmt, 0);
-  return SQLITE_OK;
-}
-
-/* Add all terms in [zText] to pendingTerms table.  If [iColumn] > 0,
-** we also store positions and offsets in the hash table using that
-** column number.
-*/
-static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
-                      const char *zText, int iColumn){
-  sqlite3_tokenizer *pTokenizer = v->pTokenizer;
-  sqlite3_tokenizer_cursor *pCursor;
-  const char *pToken;
-  int nTokenBytes;
-  int iStartOffset, iEndOffset, iPosition;
-  int rc;
-
-  rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
-  if( rc!=SQLITE_OK ) return rc;
-
-  pCursor->pTokenizer = pTokenizer;
-  while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
-                                                   &pToken, &nTokenBytes,
-                                                   &iStartOffset, &iEndOffset,
-                                                   &iPosition)) ){
-    DLCollector *p;
-    int nData;                   /* Size of doclist before our update. */
-
-    /* Positions can't be negative; we use -1 as a terminator
-     * internally.  Token can't be NULL or empty. */
-    if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){
-      rc = SQLITE_ERROR;
-      break;
-    }
-
-    p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes);
-    if( p==NULL ){
-      nData = 0;
-      p = dlcNew(iDocid, DL_DEFAULT);
-      fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
-
-      /* Overhead for our hash table entry, the key, and the value. */
-      v->nPendingData += sizeof(struct fts3HashElem)+sizeof(*p)+nTokenBytes;
-    }else{
-      nData = p->b.nData;
-      if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid);
-    }
-    if( iColumn>=0 ){
-      dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
-    }
-
-    /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
-    v->nPendingData += p->b.nData-nData;
-  }
-
-  /* TODO(shess) Check return?  Should this be able to cause errors at
-  ** this point?  Actually, same question about sqlite3_finalize(),
-  ** though one could argue that failure there means that the data is
-  ** not durable.  *ponder*
-  */
-  pTokenizer->pModule->xClose(pCursor);
-  if( SQLITE_DONE == rc ) return SQLITE_OK;
-  return rc;
-}
-
-/* Add doclists for all terms in [pValues] to pendingTerms table. */
-static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid,
-                       sqlite3_value **pValues){
-  int i;
-  for(i = 0; i < v->nColumn ; ++i){
-    char *zText = (char*)sqlite3_value_text(pValues[i]);
-    int rc = buildTerms(v, iDocid, zText, i);
-    if( rc!=SQLITE_OK ) return rc;
-  }
-  return SQLITE_OK;
-}
-
-/* Add empty doclists for all terms in the given row's content to
-** pendingTerms.
-*/
-static int deleteTerms(fulltext_vtab *v, sqlite_int64 iDocid){
-  const char **pValues;
-  int i, rc;
-
-  /* TODO(shess) Should we allow such tables at all? */
-  if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;
-
-  rc = content_select(v, iDocid, &pValues);
-  if( rc!=SQLITE_OK ) return rc;
-
-  for(i = 0 ; i < v->nColumn; ++i) {
-    rc = buildTerms(v, iDocid, pValues[i], -1);
-    if( rc!=SQLITE_OK ) break;
+    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+    sqlite3_free(zSql);
   }
-
-  freeStringArray(v->nColumn, pValues);
-  return SQLITE_OK;
-}
-
-/* TODO(shess) Refactor the code to remove this forward decl. */
-static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);
-
-/* Insert a row into the %_content table; set *piDocid to be the ID of the
-** new row.  Add doclists for terms to pendingTerms.
-*/
-static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestDocid,
-                        sqlite3_value **pValues, sqlite_int64 *piDocid){
-  int rc;
-
-  rc = content_insert(v, pRequestDocid, pValues);  /* execute an SQL INSERT */
-  if( rc!=SQLITE_OK ) return rc;
-
-  /* docid column is an alias for rowid. */
-  *piDocid = sqlite3_last_insert_rowid(v->db);
-  rc = initPendingTerms(v, *piDocid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return insertTerms(v, *piDocid, pValues);
-}
-
-/* Delete a row from the %_content table; add empty doclists for terms
-** to pendingTerms.
-*/
-static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
-  int rc = initPendingTerms(v, iRow);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = deleteTerms(v, iRow);
-  if( rc!=SQLITE_OK ) return rc;
-
-  return content_delete(v, iRow);  /* execute an SQL DELETE */
-}
-
-/* Update a row in the %_content table; add delete doclists to
-** pendingTerms for old terms not in the new data, add insert doclists
-** to pendingTerms for terms in the new data.
-*/
-static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
-                        sqlite3_value **pValues){
-  int rc = initPendingTerms(v, iRow);
   if( rc!=SQLITE_OK ) return rc;
+  pCsr->eSearch = (i16)idxNum;
 
-  /* Generate an empty doclist for each term that previously appeared in this
-   * row. */
-  rc = deleteTerms(v, iRow);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = content_update(v, pValues, iRow);  /* execute an SQL UPDATE */
-  if( rc!=SQLITE_OK ) return rc;
-
-  /* Now add positions for terms which appear in the updated row. */
-  return insertTerms(v, iRow, pValues);
-}
-
-/*******************************************************************/
-/* InteriorWriter is used to collect terms and block references into
-** interior nodes in %_segments.  See commentary at top of file for
-** format.
-*/
-
-/* How large interior nodes can grow. */
-#define INTERIOR_MAX 2048
+  if( idxNum==FTS3_DOCID_SEARCH ){
+    rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
+  }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){
+    int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
+    const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
 
-/* Minimum number of terms per interior node (except the root). This
-** prevents large terms from making the tree too skinny - must be >0
-** so that the tree always makes progress.  Note that the min tree
-** fanout will be INTERIOR_MIN_TERMS+1.
-*/
-#define INTERIOR_MIN_TERMS 7
-#if INTERIOR_MIN_TERMS<1
-# error INTERIOR_MIN_TERMS must be greater than 0.
-#endif
-
-/* ROOT_MAX controls how much data is stored inline in the segment
-** directory.
-*/
-/* TODO(shess) Push ROOT_MAX down to whoever is writing things.  It's
-** only here so that interiorWriterRootInfo() and leafWriterRootInfo()
-** can both see it, but if the caller passed it in, we wouldn't even
-** need a define.
-*/
-#define ROOT_MAX 1024
-#if ROOT_MAX<VARINT_MAX*2
-# error ROOT_MAX must have enough space for a header.
-#endif
-
-/* InteriorBlock stores a linked-list of interior blocks while a lower
-** layer is being constructed.
-*/
-typedef struct InteriorBlock {
-  DataBuffer term;           /* Leftmost term in block's subtree. */
-  DataBuffer data;           /* Accumulated data for the block. */
-  struct InteriorBlock *next;
-} InteriorBlock;
-
-static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock,
-                                       const char *pTerm, int nTerm){
-  InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock));
-  char c[VARINT_MAX+VARINT_MAX];
-  int n;
-
-  if( block ){
-    memset(block, 0, sizeof(*block));
-    dataBufferInit(&block->term, 0);
-    dataBufferReplace(&block->term, pTerm, nTerm);
-
-    n = fts3PutVarint(c, iHeight);
-    n += fts3PutVarint(c+n, iChildBlock);
-    dataBufferInit(&block->data, INTERIOR_MAX);
-    dataBufferReplace(&block->data, c, n);
-  }
-  return block;
-}
-
-#ifndef NDEBUG
-/* Verify that the data is readable as an interior node. */
-static void interiorBlockValidate(InteriorBlock *pBlock){
-  const char *pData = pBlock->data.pData;
-  int nData = pBlock->data.nData;
-  int n, iDummy;
-  sqlite_int64 iBlockid;
-
-  assert( nData>0 );
-  assert( pData!=0 );
-  assert( pData+nData>pData );
-
-  /* Must lead with height of node as a varint(n), n>0 */
-  n = fts3GetVarint32(pData, &iDummy);
-  assert( n>0 );
-  assert( iDummy>0 );
-  assert( n<nData );
-  pData += n;
-  nData -= n;
-
-  /* Must contain iBlockid. */
-  n = fts3GetVarint(pData, &iBlockid);
-  assert( n>0 );
-  assert( n<=nData );
-  pData += n;
-  nData -= n;
-
-  /* Zero or more terms of positive length */
-  if( nData!=0 ){
-    /* First term is not delta-encoded. */
-    n = fts3GetVarint32(pData, &iDummy);
-    assert( n>0 );
-    assert( iDummy>0 );
-    assert( n+iDummy>0);
-    assert( n+iDummy<=nData );
-    pData += n+iDummy;
-    nData -= n+iDummy;
-
-    /* Following terms delta-encoded. */
-    while( nData!=0 ){
-      /* Length of shared prefix. */
-      n = fts3GetVarint32(pData, &iDummy);
-      assert( n>0 );
-      assert( iDummy>=0 );
-      assert( n<nData );
-      pData += n;
-      nData -= n;
-
-      /* Length and data of distinct suffix. */
-      n = fts3GetVarint32(pData, &iDummy);
-      assert( n>0 );
-      assert( iDummy>0 );
-      assert( n+iDummy>0);
-      assert( n+iDummy<=nData );
-      pData += n+iDummy;
-      nData -= n+iDummy;
+    if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
+      return SQLITE_NOMEM;
     }
-  }
-}
-#define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x)
-#else
-#define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 )
-#endif
-
-typedef struct InteriorWriter {
-  int iHeight;                   /* from 0 at leaves. */
-  InteriorBlock *first, *last;
-  struct InteriorWriter *parentWriter;
 
-  DataBuffer term;               /* Last term written to block "last". */
-  sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */
-#ifndef NDEBUG
-  sqlite_int64 iLastChildBlock;  /* for consistency checks. */
-#endif
-} InteriorWriter;
-
-/* Initialize an interior node where pTerm[nTerm] marks the leftmost
-** term in the tree.  iChildBlock is the leftmost child block at the
-** next level down the tree.
-*/
-static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm,
-                               sqlite_int64 iChildBlock,
-                               InteriorWriter *pWriter){
-  InteriorBlock *block;
-  assert( iHeight>0 );
-  CLEAR(pWriter);
-
-  pWriter->iHeight = iHeight;
-  pWriter->iOpeningChildBlock = iChildBlock;
-#ifndef NDEBUG
-  pWriter->iLastChildBlock = iChildBlock;
-#endif
-  block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm);
-  pWriter->last = pWriter->first = block;
-  ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
-  dataBufferInit(&pWriter->term, 0);
-}
-
-/* Append the child node rooted at iChildBlock to the interior node,
-** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree.
-*/
-static void interiorWriterAppend(InteriorWriter *pWriter,
-                                 const char *pTerm, int nTerm,
-                                 sqlite_int64 iChildBlock){
-  char c[VARINT_MAX+VARINT_MAX];
-  int n, nPrefix = 0;
-
-  ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
-
-  /* The first term written into an interior node is actually
-  ** associated with the second child added (the first child was added
-  ** in interiorWriterInit, or in the if clause at the bottom of this
-  ** function).  That term gets encoded straight up, with nPrefix left
-  ** at 0.
-  */
-  if( pWriter->term.nData==0 ){
-    n = fts3PutVarint(c, nTerm);
-  }else{
-    while( nPrefix<pWriter->term.nData &&
-           pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
-      nPrefix++;
+    rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, 
+        iCol, zQuery, -1, &pCsr->pExpr
+    );
+    if( rc!=SQLITE_OK ){
+      if( rc==SQLITE_ERROR ){
+        p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]",
+                                          zQuery);
+      }
+      return rc;
     }
 
-    n = fts3PutVarint(c, nPrefix);
-    n += fts3PutVarint(c+n, nTerm-nPrefix);
-  }
-
-#ifndef NDEBUG
-  pWriter->iLastChildBlock++;
-#endif
-  assert( pWriter->iLastChildBlock==iChildBlock );
-
-  /* Overflow to a new block if the new term makes the current block
-  ** too big, and the current block already has enough terms.
-  */
-  if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX &&
-      iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){
-    pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock,
-                                           pTerm, nTerm);
-    pWriter->last = pWriter->last->next;
-    pWriter->iOpeningChildBlock = iChildBlock;
-    dataBufferReset(&pWriter->term);
-  }else{
-    dataBufferAppend2(&pWriter->last->data, c, n,
-                      pTerm+nPrefix, nTerm-nPrefix);
-    dataBufferReplace(&pWriter->term, pTerm, nTerm);
-  }
-  ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
-}
-
-/* Free the space used by pWriter, including the linked-list of
-** InteriorBlocks, and parentWriter, if present.
-*/
-static int interiorWriterDestroy(InteriorWriter *pWriter){
-  InteriorBlock *block = pWriter->first;
-
-  while( block!=NULL ){
-    InteriorBlock *b = block;
-    block = block->next;
-    dataBufferDestroy(&b->term);
-    dataBufferDestroy(&b->data);
-    sqlite3_free(b);
-  }
-  if( pWriter->parentWriter!=NULL ){
-    interiorWriterDestroy(pWriter->parentWriter);
-    sqlite3_free(pWriter->parentWriter);
-  }
-  dataBufferDestroy(&pWriter->term);
-  SCRAMBLE(pWriter);
-  return SQLITE_OK;
-}
-
-/* If pWriter can fit entirely in ROOT_MAX, return it as the root info
-** directly, leaving *piEndBlockid unchanged.  Otherwise, flush
-** pWriter to %_segments, building a new layer of interior nodes, and
-** recursively ask for their root into.
-*/
-static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter,
-                                  char **ppRootInfo, int *pnRootInfo,
-                                  sqlite_int64 *piEndBlockid){
-  InteriorBlock *block = pWriter->first;
-  sqlite_int64 iBlockid = 0;
-  int rc;
-
-  /* If we can fit the segment inline */
-  if( block==pWriter->last && block->data.nData<ROOT_MAX ){
-    *ppRootInfo = block->data.pData;
-    *pnRootInfo = block->data.nData;
-    return SQLITE_OK;
-  }
-
-  /* Flush the first block to %_segments, and create a new level of
-  ** interior node.
-  */
-  ASSERT_VALID_INTERIOR_BLOCK(block);
-  rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
-  if( rc!=SQLITE_OK ) return rc;
-  *piEndBlockid = iBlockid;
-
-  pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter));
-  interiorWriterInit(pWriter->iHeight+1,
-                     block->term.pData, block->term.nData,
-                     iBlockid, pWriter->parentWriter);
-
-  /* Flush additional blocks and append to the higher interior
-  ** node.
-  */
-  for(block=block->next; block!=NULL; block=block->next){
-    ASSERT_VALID_INTERIOR_BLOCK(block);
-    rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
+    rc = sqlite3Fts3ReadLock(p);
     if( rc!=SQLITE_OK ) return rc;
-    *piEndBlockid = iBlockid;
-
-    interiorWriterAppend(pWriter->parentWriter,
-                         block->term.pData, block->term.nData, iBlockid);
-  }
-
-  /* Parent node gets the chance to be the root. */
-  return interiorWriterRootInfo(v, pWriter->parentWriter,
-                                ppRootInfo, pnRootInfo, piEndBlockid);
-}
-
-/****************************************************************/
-/* InteriorReader is used to read off the data from an interior node
-** (see comment at top of file for the format).
-*/
-typedef struct InteriorReader {
-  const char *pData;
-  int nData;
-
-  DataBuffer term;          /* previous term, for decoding term delta. */
 
-  sqlite_int64 iBlockid;
-} InteriorReader;
-
-static void interiorReaderDestroy(InteriorReader *pReader){
-  dataBufferDestroy(&pReader->term);
-  SCRAMBLE(pReader);
-}
-
-static int interiorReaderInit(const char *pData, int nData,
-                              InteriorReader *pReader){
-  int n, nTerm;
-
-  /* These conditions are checked and met by the callers. */
-  assert( nData>0 );
-  assert( pData[0]!='\0' );
-
-  CLEAR(pReader);
-
-  /* Decode the base blockid, and set the cursor to the first term. */
-  n = fts3GetVarintSafe(pData+1, &pReader->iBlockid, nData-1);
-  if( !n ) return SQLITE_CORRUPT_BKPT;
-  pReader->pData = pData+1+n;
-  pReader->nData = nData-(1+n);
-
-  /* A single-child interior node (such as when a leaf node was too
-  ** large for the segment directory) won't have any terms.
-  ** Otherwise, decode the first term.
-  */
-  if( pReader->nData==0 ){
-    dataBufferInit(&pReader->term, 0);
-  }else{
-    n = fts3GetVarint32Safe(pReader->pData, &nTerm, pReader->nData);
-    if( !n || nTerm<0 || nTerm>pReader->nData-n) return SQLITE_CORRUPT_BKPT;
-    dataBufferInit(&pReader->term, nTerm);
-    dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
-    pReader->pData += n+nTerm;
-    pReader->nData -= n+nTerm;
+    rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
+    pCsr->pNextId = pCsr->aDoclist;
+    pCsr->iPrevId = 0;
   }
-  return SQLITE_OK;
-}
-
-static int interiorReaderAtEnd(InteriorReader *pReader){
-  return pReader->term.nData<=0;
-}
-
-static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
-  return pReader->iBlockid;
-}
 
-static int interiorReaderTermBytes(InteriorReader *pReader){
-  assert( !interiorReaderAtEnd(pReader) );
-  return pReader->term.nData;
-}
-static const char *interiorReaderTerm(InteriorReader *pReader){
-  assert( !interiorReaderAtEnd(pReader) );
-  return pReader->term.pData;
-}
-
-/* Step forward to the next term in the node. */
-static int interiorReaderStep(InteriorReader *pReader){
-  assert( !interiorReaderAtEnd(pReader) );
-
-  /* If the last term has been read, signal eof, else construct the
-  ** next term.
-  */
-  if( pReader->nData==0 ){
-    dataBufferReset(&pReader->term);
-  }else{
-    int n, nPrefix, nSuffix;
-
-    n = fts3GetVarint32Safe(pReader->pData, &nPrefix, pReader->nData);
-    if( !n ) return SQLITE_CORRUPT_BKPT;
-    pReader->nData -= n;
-    pReader->pData += n;
-    n = fts3GetVarint32Safe(pReader->pData, &nSuffix, pReader->nData);
-    if( !n ) return SQLITE_CORRUPT_BKPT;
-    pReader->nData -= n;
-    pReader->pData += n;
-    if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT;
-    if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT;
-
-    /* Truncate the current term and append suffix data. */
-    pReader->term.nData = nPrefix;
-    dataBufferAppend(&pReader->term, pReader->pData, nSuffix);
-
-    pReader->pData += nSuffix;
-    pReader->nData -= nSuffix;
-  }
-  pReader->iBlockid++;
-  return SQLITE_OK;
-}
-
-/* Compare the current term to pTerm[nTerm], returning strcmp-style
-** results.  If isPrefix, equality means equal through nTerm bytes.
-*/
-static int interiorReaderTermCmp(InteriorReader *pReader,
-                                 const char *pTerm, int nTerm, int isPrefix){
-  const char *pReaderTerm = interiorReaderTerm(pReader);
-  int nReaderTerm = interiorReaderTermBytes(pReader);
-  int c, n = nReaderTerm<nTerm ? nReaderTerm : nTerm;
-
-  if( n==0 ){
-    if( nReaderTerm>0 ) return -1;
-    if( nTerm>0 ) return 1;
-    return 0;
-  }
-
-  c = memcmp(pReaderTerm, pTerm, n);
-  if( c!=0 ) return c;
-  if( isPrefix && n==nTerm ) return 0;
-  return nReaderTerm - nTerm;
+  if( rc!=SQLITE_OK ) return rc;
+  return fts3NextMethod(pCursor);
 }
 
-/****************************************************************/
-/* LeafWriter is used to collect terms and associated doclist data
-** into leaf blocks in %_segments (see top of file for format info).
-** Expected usage is:
-**
-** LeafWriter writer;
-** leafWriterInit(0, 0, &writer);
-** while( sorted_terms_left_to_process ){
-**   // data is doclist data for that term.
-**   rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData);
-**   if( rc!=SQLITE_OK ) goto err;
-** }
-** rc = leafWriterFinalize(v, &writer);
-**err:
-** leafWriterDestroy(&writer);
-** return rc;
-**
-** leafWriterStep() may write a collected leaf out to %_segments.
-** leafWriterFinalize() finishes writing any buffered data and stores
-** a root node in %_segdir.  leafWriterDestroy() frees all buffers and
-** InteriorWriters allocated as part of writing this segment.
-**
-** TODO(shess) Document leafWriterStepMerge().
+/* 
+** This is the xEof method of the virtual table. SQLite calls this 
+** routine to find out if it has reached the end of a result set.
 */
-
-/* Put terms with data this big in their own block. */
-#define STANDALONE_MIN 1024
-
-/* Keep leaf blocks below this size. */
-#define LEAF_MAX 2048
-
-typedef struct LeafWriter {
-  int iLevel;
-  int idx;
-  sqlite_int64 iStartBlockid;     /* needed to create the root info */
-  sqlite_int64 iEndBlockid;       /* when we're done writing. */
-
-  DataBuffer term;                /* previous encoded term */
-  DataBuffer data;                /* encoding buffer */
-
-  /* bytes of first term in the current node which distinguishes that
-  ** term from the last term of the previous node.
-  */
-  int nTermDistinct;
-
-  InteriorWriter parentWriter;    /* if we overflow */
-  int has_parent;
-} LeafWriter;
-
-static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){
-  CLEAR(pWriter);
-  pWriter->iLevel = iLevel;
-  pWriter->idx = idx;
-
-  dataBufferInit(&pWriter->term, 32);
-
-  /* Start out with a reasonably sized block, though it can grow. */
-  dataBufferInit(&pWriter->data, LEAF_MAX);
-}
-
-#ifndef NDEBUG
-/* Verify that the data is readable as a leaf node. */
-static void leafNodeValidate(const char *pData, int nData){
-  int n, iDummy;
-
-  if( nData==0 ) return;
-  assert( nData>0 );
-  assert( pData!=0 );
-  assert( pData+nData>pData );
-
-  /* Must lead with a varint(0) */
-  n = fts3GetVarint32(pData, &iDummy);
-  assert( iDummy==0 );
-  assert( n>0 );
-  assert( n<nData );
-  pData += n;
-  nData -= n;
-
-  /* Leading term length and data must fit in buffer. */
-  n = fts3GetVarint32(pData, &iDummy);
-  assert( n>0 );
-  assert( iDummy>0 );
-  assert( n+iDummy>0 );
-  assert( n+iDummy<nData );
-  pData += n+iDummy;
-  nData -= n+iDummy;
-
-  /* Leading term's doclist length and data must fit. */
-  n = fts3GetVarint32(pData, &iDummy);
-  assert( n>0 );
-  assert( iDummy>0 );
-  assert( n+iDummy>0 );
-  assert( n+iDummy<=nData );
-  ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
-  pData += n+iDummy;
-  nData -= n+iDummy;
-
-  /* Verify that trailing terms and doclists also are readable. */
-  while( nData!=0 ){
-    n = fts3GetVarint32(pData, &iDummy);
-    assert( n>0 );
-    assert( iDummy>=0 );
-    assert( n<nData );
-    pData += n;
-    nData -= n;
-    n = fts3GetVarint32(pData, &iDummy);
-    assert( n>0 );
-    assert( iDummy>0 );
-    assert( n+iDummy>0 );
-    assert( n+iDummy<nData );
-    pData += n+iDummy;
-    nData -= n+iDummy;
-
-    n = fts3GetVarint32(pData, &iDummy);
-    assert( n>0 );
-    assert( iDummy>0 );
-    assert( n+iDummy>0 );
-    assert( n+iDummy<=nData );
-    ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
-    pData += n+iDummy;
-    nData -= n+iDummy;
-  }
+static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
+  return ((Fts3Cursor *)pCursor)->isEof;
 }
-#define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n)
-#else
-#define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 )
-#endif
 
-/* Flush the current leaf node to %_segments, and adding the resulting
-** blockid and the starting term to the interior node which will
-** contain it.
+/* 
+** This is the xRowid method. The SQLite core calls this routine to
+** retrieve the rowid for the current row of the result set. fts3
+** exposes %_content.docid as the rowid for the virtual table. The
+** rowid should be written to *pRowid.
 */
-static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter,
-                                   int iData, int nData){
-  sqlite_int64 iBlockid = 0;
-  const char *pStartingTerm;
-  int nStartingTerm, rc, n;
-
-  /* Must have the leading varint(0) flag, plus at least some
-  ** valid-looking data.
-  */
-  assert( nData>2 );
-  assert( iData>=0 );
-  assert( iData+nData<=pWriter->data.nData );
-  ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData);
-
-  rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid);
-  if( rc!=SQLITE_OK ) return rc;
-  assert( iBlockid!=0 );
-
-  /* Reconstruct the first term in the leaf for purposes of building
-  ** the interior node.
-  */
-  n = fts3GetVarint32(pWriter->data.pData+iData+1, &nStartingTerm);
-  pStartingTerm = pWriter->data.pData+iData+1+n;
-  assert( pWriter->data.nData>iData+1+n+nStartingTerm );
-  assert( pWriter->nTermDistinct>0 );
-  assert( pWriter->nTermDistinct<=nStartingTerm );
-  nStartingTerm = pWriter->nTermDistinct;
-
-  if( pWriter->has_parent ){
-    interiorWriterAppend(&pWriter->parentWriter,
-                         pStartingTerm, nStartingTerm, iBlockid);
-  }else{
-    interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid,
-                       &pWriter->parentWriter);
-    pWriter->has_parent = 1;
-  }
-
-  /* Track the span of this segment's leaf nodes. */
-  if( pWriter->iEndBlockid==0 ){
-    pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid;
+static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+  Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
+  if( pCsr->aDoclist ){
+    *pRowid = pCsr->iPrevId;
   }else{
-    pWriter->iEndBlockid++;
-    assert( iBlockid==pWriter->iEndBlockid );
+    *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
   }
-
   return SQLITE_OK;
 }
-static int leafWriterFlush(fulltext_vtab *v, LeafWriter *pWriter){
-  int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData);
-  if( rc!=SQLITE_OK ) return rc;
-
-  /* Re-initialize the output buffer. */
-  dataBufferReset(&pWriter->data);
 
-  return SQLITE_OK;
-}
-
-/* Fetch the root info for the segment.  If the entire leaf fits
-** within ROOT_MAX, then it will be returned directly, otherwise it
-** will be flushed and the root info will be returned from the
-** interior node.  *piEndBlockid is set to the blockid of the last
-** interior or leaf node written to disk (0 if none are written at
-** all).
-*/
-static int leafWriterRootInfo(fulltext_vtab *v, LeafWriter *pWriter,
-                              char **ppRootInfo, int *pnRootInfo,
-                              sqlite_int64 *piEndBlockid){
-  /* we can fit the segment entirely inline */
-  if( !pWriter->has_parent && pWriter->data.nData<ROOT_MAX ){
-    *ppRootInfo = pWriter->data.pData;
-    *pnRootInfo = pWriter->data.nData;
-    *piEndBlockid = 0;
-    return SQLITE_OK;
-  }
-
-  /* Flush remaining leaf data. */
-  if( pWriter->data.nData>0 ){
-    int rc = leafWriterFlush(v, pWriter);
-    if( rc!=SQLITE_OK ) return rc;
-  }
-
-  /* We must have flushed a leaf at some point. */
-  assert( pWriter->has_parent );
-
-  /* Tenatively set the end leaf blockid as the end blockid.  If the
-  ** interior node can be returned inline, this will be the final
-  ** blockid, otherwise it will be overwritten by
-  ** interiorWriterRootInfo().
-  */
-  *piEndBlockid = pWriter->iEndBlockid;
-
-  return interiorWriterRootInfo(v, &pWriter->parentWriter,
-                                ppRootInfo, pnRootInfo, piEndBlockid);
-}
-
-/* Collect the rootInfo data and store it into the segment directory.
-** This has the effect of flushing the segment's leaf data to
-** %_segments, and also flushing any interior nodes to %_segments.
+/* 
+** This is the xColumn method, called by SQLite to request a value from
+** the row that the supplied cursor currently points to.
 */
-static int leafWriterFinalize(fulltext_vtab *v, LeafWriter *pWriter){
-  sqlite_int64 iEndBlockid;
-  char *pRootInfo;
-  int rc, nRootInfo;
-
-  rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  /* Don't bother storing an entirely empty segment. */
-  if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK;
-
-  return segdir_set(v, pWriter->iLevel, pWriter->idx,
-                    pWriter->iStartBlockid, pWriter->iEndBlockid,
-                    iEndBlockid, pRootInfo, nRootInfo);
-}
+static int fts3ColumnMethod(
+  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
+  sqlite3_context *pContext,      /* Context for sqlite3_result_xxx() calls */
+  int iCol                        /* Index of column to read value from */
+){
+  int rc;                         /* Return Code */
+  Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
+  Fts3Table *p = (Fts3Table *)pCursor->pVtab;
 
-static void leafWriterDestroy(LeafWriter *pWriter){
-  if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter);
-  dataBufferDestroy(&pWriter->term);
-  dataBufferDestroy(&pWriter->data);
-}
+  /* The column value supplied by SQLite must be in range. */
+  assert( iCol>=0 && iCol<=p->nColumn+1 );
 
-/* Encode a term into the leafWriter, delta-encoding as appropriate.
-** Returns the length of the new term which distinguishes it from the
-** previous term, which can be used to set nTermDistinct when a node
-** boundary is crossed.
-*/
-static int leafWriterEncodeTerm(LeafWriter *pWriter,
-                                const char *pTerm, int nTerm){
-  char c[VARINT_MAX+VARINT_MAX];
-  int n, nPrefix = 0;
-
-  assert( nTerm>0 );
-  while( nPrefix<pWriter->term.nData &&
-         pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
-    nPrefix++;
-    /* Failing this implies that the terms weren't in order. */
-    assert( nPrefix<nTerm );
-  }
-
-  if( pWriter->data.nData==0 ){
-    /* Encode the node header and leading term as:
-    **  varint(0)
-    **  varint(nTerm)
-    **  char pTerm[nTerm]
+  if( iCol==p->nColumn+1 ){
+    /* This call is a request for the "docid" column. Since "docid" is an 
+    ** alias for "rowid", use the xRowid() method to obtain the value.
     */
-    n = fts3PutVarint(c, '\0');
-    n += fts3PutVarint(c+n, nTerm);
-    dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm);
-  }else{
-    /* Delta-encode the term as:
-    **  varint(nPrefix)
-    **  varint(nSuffix)
-    **  char pTermSuffix[nSuffix]
+    sqlite3_int64 iRowid;
+    rc = fts3RowidMethod(pCursor, &iRowid);
+    sqlite3_result_int64(pContext, iRowid);
+  }else if( iCol==p->nColumn ){
+    /* The extra column whose name is the same as the table.
+    ** Return a blob which is a pointer to the cursor.
     */
-    n = fts3PutVarint(c, nPrefix);
-    n += fts3PutVarint(c+n, nTerm-nPrefix);
-    dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix);
-  }
-  dataBufferReplace(&pWriter->term, pTerm, nTerm);
-
-  return nPrefix+1;
-}
-
-/* Used to avoid a memmove when a large amount of doclist data is in
-** the buffer.  This constructs a node and term header before
-** iDoclistData and flushes the resulting complete node using
-** leafWriterInternalFlush().
-*/
-static int leafWriterInlineFlush(fulltext_vtab *v, LeafWriter *pWriter,
-                                 const char *pTerm, int nTerm,
-                                 int iDoclistData){
-  char c[VARINT_MAX+VARINT_MAX];
-  int iData, n = fts3PutVarint(c, 0);
-  n += fts3PutVarint(c+n, nTerm);
-
-  /* There should always be room for the header.  Even if pTerm shared
-  ** a substantial prefix with the previous term, the entire prefix
-  ** could be constructed from earlier data in the doclist, so there
-  ** should be room.
-  */
-  assert( iDoclistData>=n+nTerm );
-
-  iData = iDoclistData-(n+nTerm);
-  memcpy(pWriter->data.pData+iData, c, n);
-  memcpy(pWriter->data.pData+iData+n, pTerm, nTerm);
-
-  return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData);
-}
-
-/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
-** %_segments.
-*/
-static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter,
-                               const char *pTerm, int nTerm,
-                               DLReader *pReaders, int nReaders){
-  char c[VARINT_MAX+VARINT_MAX];
-  int iTermData = pWriter->data.nData, iDoclistData;
-  int i, nData, n, nActualData, nActual, rc, nTermDistinct;
-
-  ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
-  nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm);
-
-  /* Remember nTermDistinct if opening a new node. */
-  if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct;
-
-  iDoclistData = pWriter->data.nData;
-
-  /* Estimate the length of the merged doclist so we can leave space
-  ** to encode it.
-  */
-  for(i=0, nData=0; i<nReaders; i++){
-    nData += dlrAllDataBytes(&pReaders[i]);
-  }
-  n = fts3PutVarint(c, nData);
-  dataBufferAppend(&pWriter->data, c, n);
-
-  rc = docListMerge(&pWriter->data, pReaders, nReaders);
-  if( rc!=SQLITE_OK ) return rc;
-  ASSERT_VALID_DOCLIST(DL_DEFAULT,
-                       pWriter->data.pData+iDoclistData+n,
-                       pWriter->data.nData-iDoclistData-n, NULL);
-
-  /* The actual amount of doclist data at this point could be smaller
-  ** than the length we encoded.  Additionally, the space required to
-  ** encode this length could be smaller.  For small doclists, this is
-  ** not a big deal, we can just use memmove() to adjust things.
-  */
-  nActualData = pWriter->data.nData-(iDoclistData+n);
-  nActual = fts3PutVarint(c, nActualData);
-  assert( nActualData<=nData );
-  assert( nActual<=n );
-
-  /* If the new doclist is big enough for force a standalone leaf
-  ** node, we can immediately flush it inline without doing the
-  ** memmove().
-  */
-  /* TODO(shess) This test matches leafWriterStep(), which does this
-  ** test before it knows the cost to varint-encode the term and
-  ** doclist lengths.  At some point, change to
-  ** pWriter->data.nData-iTermData>STANDALONE_MIN.
-  */
-  if( nTerm+nActualData>STANDALONE_MIN ){
-    /* Push leaf node from before this term. */
-    if( iTermData>0 ){
-      rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
-      if( rc!=SQLITE_OK ) return rc;
-
-      pWriter->nTermDistinct = nTermDistinct;
+    sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
+    rc = SQLITE_OK;
+  }else{
+    rc = fts3CursorSeek(0, pCsr);
+    if( rc==SQLITE_OK ){
+      sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
     }
-
-    /* Fix the encoded doclist length. */
-    iDoclistData += n - nActual;
-    memcpy(pWriter->data.pData+iDoclistData, c, nActual);
-
-    /* Push the standalone leaf node. */
-    rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData);
-    if( rc!=SQLITE_OK ) return rc;
-
-    /* Leave the node empty. */
-    dataBufferReset(&pWriter->data);
-
-    return rc;
-  }
-
-  /* At this point, we know that the doclist was small, so do the
-  ** memmove if indicated.
-  */
-  if( nActual<n ){
-    memmove(pWriter->data.pData+iDoclistData+nActual,
-            pWriter->data.pData+iDoclistData+n,
-            pWriter->data.nData-(iDoclistData+n));
-    pWriter->data.nData -= n-nActual;
-  }
-
-  /* Replace written length with actual length. */
-  memcpy(pWriter->data.pData+iDoclistData, c, nActual);
-
-  /* If the node is too large, break things up. */
-  /* TODO(shess) This test matches leafWriterStep(), which does this
-  ** test before it knows the cost to varint-encode the term and
-  ** doclist lengths.  At some point, change to
-  ** pWriter->data.nData>LEAF_MAX.
-  */
-  if( iTermData+nTerm+nActualData>LEAF_MAX ){
-    /* Flush out the leading data as a node */
-    rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
-    if( rc!=SQLITE_OK ) return rc;
-
-    pWriter->nTermDistinct = nTermDistinct;
-
-    /* Rebuild header using the current term */
-    n = fts3PutVarint(pWriter->data.pData, 0);
-    n += fts3PutVarint(pWriter->data.pData+n, nTerm);
-    memcpy(pWriter->data.pData+n, pTerm, nTerm);
-    n += nTerm;
-
-    /* There should always be room, because the previous encoding
-    ** included all data necessary to construct the term.
-    */
-    assert( n<iDoclistData );
-    /* So long as STANDALONE_MIN is half or less of LEAF_MAX, the
-    ** following memcpy() is safe (as opposed to needing a memmove).
-    */
-    assert( 2*STANDALONE_MIN<=LEAF_MAX );
-    assert( n+pWriter->data.nData-iDoclistData<iDoclistData );
-    memcpy(pWriter->data.pData+n,
-           pWriter->data.pData+iDoclistData,
-           pWriter->data.nData-iDoclistData);
-    pWriter->data.nData -= iDoclistData-n;
   }
-  ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
-
-  return SQLITE_OK;
-}
-
-/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
-** %_segments.
-*/
-/* TODO(shess) Revise writeZeroSegment() so that doclists are
-** constructed directly in pWriter->data.
-*/
-static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter,
-                          const char *pTerm, int nTerm,
-                          const char *pData, int nData){
-  int rc;
-  DLReader reader;
-
-  rc = dlrInit(&reader, DL_DEFAULT, pData, nData);
-  if( rc!=SQLITE_OK ) return rc;
-  rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
-  dlrDestroy(&reader);
-
   return rc;
 }
 
-
-/****************************************************************/
-/* LeafReader is used to iterate over an individual leaf node. */
-typedef struct LeafReader {
-  DataBuffer term;          /* copy of current term. */
-
-  const char *pData;        /* data for current term. */
-  int nData;
-} LeafReader;
-
-static void leafReaderDestroy(LeafReader *pReader){
-  dataBufferDestroy(&pReader->term);
-  SCRAMBLE(pReader);
-}
-
-static int leafReaderAtEnd(LeafReader *pReader){
-  return pReader->nData<=0;
-}
-
-/* Access the current term. */
-static int leafReaderTermBytes(LeafReader *pReader){
-  return pReader->term.nData;
-}
-static const char *leafReaderTerm(LeafReader *pReader){
-  assert( pReader->term.nData>0 );
-  return pReader->term.pData;
-}
-
-/* Access the doclist data for the current term. */
-static int leafReaderDataBytes(LeafReader *pReader){
-  int nData;
-  assert( pReader->term.nData>0 );
-  fts3GetVarint32(pReader->pData, &nData);
-  return nData;
-}
-static const char *leafReaderData(LeafReader *pReader){
-  int n, nData;
-  assert( pReader->term.nData>0 );
-  n = fts3GetVarint32Safe(pReader->pData, &nData, pReader->nData);
-  if( !n || nData>pReader->nData-n ) return NULL;
-  return pReader->pData+n;
-}
-
-static int leafReaderInit(const char *pData, int nData,
-                          LeafReader *pReader){
-  int nTerm, n;
-
-  /* All callers check this precondition. */
-  assert( nData>0 );
-  assert( pData[0]=='\0' );
-
-  CLEAR(pReader);
-
-  /* Read the first term, skipping the header byte. */
-  n = fts3GetVarint32Safe(pData+1, &nTerm, nData-1);
-  if( !n || nTerm<0 || nTerm>nData-1-n ) return SQLITE_CORRUPT_BKPT;
-  dataBufferInit(&pReader->term, nTerm);
-  dataBufferReplace(&pReader->term, pData+1+n, nTerm);
-
-  /* Position after the first term. */
-  pReader->pData = pData+1+n+nTerm;
-  pReader->nData = nData-1-n-nTerm;
-  return SQLITE_OK;
-}
-
-/* Step the reader forward to the next term. */
-static int leafReaderStep(LeafReader *pReader){
-  int n, nData, nPrefix, nSuffix;
-  assert( !leafReaderAtEnd(pReader) );
-
-  /* Skip previous entry's data block. */
-  n = fts3GetVarint32Safe(pReader->pData, &nData, pReader->nData);
-  if( !n || nData<0 || nData>pReader->nData-n ) return SQLITE_CORRUPT_BKPT;
-  pReader->pData += n+nData;
-  pReader->nData -= n+nData;
-
-  if( !leafReaderAtEnd(pReader) ){
-    /* Construct the new term using a prefix from the old term plus a
-    ** suffix from the leaf data.
-    */
-    n = fts3GetVarint32Safe(pReader->pData, &nPrefix, pReader->nData);
-    if( !n ) return SQLITE_CORRUPT_BKPT;
-    pReader->nData -= n;
-    pReader->pData += n;
-    n = fts3GetVarint32Safe(pReader->pData, &nSuffix, pReader->nData);
-    if( !n ) return SQLITE_CORRUPT_BKPT;
-    pReader->nData -= n;
-    pReader->pData += n;
-    if( nSuffix<0 || nSuffix>pReader->nData ) return SQLITE_CORRUPT_BKPT;
-    if( nPrefix<0 || nPrefix>pReader->term.nData ) return SQLITE_CORRUPT_BKPT;
-    pReader->term.nData = nPrefix;
-    dataBufferAppend(&pReader->term, pReader->pData, nSuffix);
-
-    pReader->pData += nSuffix;
-    pReader->nData -= nSuffix;
-  }
-  return SQLITE_OK;
-}
-
-/* strcmp-style comparison of pReader's current term against pTerm.
-** If isPrefix, equality means equal through nTerm bytes.
-*/
-static int leafReaderTermCmp(LeafReader *pReader,
-                             const char *pTerm, int nTerm, int isPrefix){
-  int c, n = pReader->term.nData<nTerm ? pReader->term.nData : nTerm;
-  if( n==0 ){
-    if( pReader->term.nData>0 ) return -1;
-    if(nTerm>0 ) return 1;
-    return 0;
-  }
-
-  c = memcmp(pReader->term.pData, pTerm, n);
-  if( c!=0 ) return c;
-  if( isPrefix && n==nTerm ) return 0;
-  return pReader->term.nData - nTerm;
-}
-
-
-/****************************************************************/
-/* LeavesReader wraps LeafReader to allow iterating over the entire
-** leaf layer of the tree.
-*/
-typedef struct LeavesReader {
-  int idx;                  /* Index within the segment. */
-
-  sqlite3_stmt *pStmt;      /* Statement we're streaming leaves from. */
-  int eof;                  /* we've seen SQLITE_DONE from pStmt. */
-
-  LeafReader leafReader;    /* reader for the current leaf. */
-  DataBuffer rootData;      /* root data for inline. */
-} LeavesReader;
-
-/* Access the current term. */
-static int leavesReaderTermBytes(LeavesReader *pReader){
-  assert( !pReader->eof );
-  return leafReaderTermBytes(&pReader->leafReader);
-}
-static const char *leavesReaderTerm(LeavesReader *pReader){
-  assert( !pReader->eof );
-  return leafReaderTerm(&pReader->leafReader);
-}
-
-/* Access the doclist data for the current term. */
-static int leavesReaderDataBytes(LeavesReader *pReader){
-  assert( !pReader->eof );
-  return leafReaderDataBytes(&pReader->leafReader);
-}
-static const char *leavesReaderData(LeavesReader *pReader){
-  assert( !pReader->eof );
-  return leafReaderData(&pReader->leafReader);
-}
-
-static int leavesReaderAtEnd(LeavesReader *pReader){
-  return pReader->eof;
-}
-
-/* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus
-** leaving the statement handle open, which locks the table.
-*/
-/* TODO(shess) This "solution" is not satisfactory.  Really, there
-** should be check-in function for all statement handles which
-** arranges to call sqlite3_reset().  This most likely will require
-** modification to control flow all over the place, though, so for now
-** just punt.
-**
-** Note the the current system assumes that segment merges will run to
-** completion, which is why this particular probably hasn't arisen in
-** this case.  Probably a brittle assumption.
-*/
-static int leavesReaderReset(LeavesReader *pReader){
-  return sqlite3_reset(pReader->pStmt);
-}
-
-static void leavesReaderDestroy(LeavesReader *pReader){
-  /* If idx is -1, that means we're using a non-cached statement
-  ** handle in the optimize() case, so we need to release it.
-  */
-  if( pReader->pStmt!=NULL && pReader->idx==-1 ){
-    sqlite3_finalize(pReader->pStmt);
-  }
-  leafReaderDestroy(&pReader->leafReader);
-  dataBufferDestroy(&pReader->rootData);
-  SCRAMBLE(pReader);
+/* 
+** This function is the implementation of the xUpdate callback used by 
+** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
+** inserted, updated or deleted.
+*/
+static int fts3UpdateMethod(
+  sqlite3_vtab *pVtab,            /* Virtual table handle */
+  int nArg,                       /* Size of argument array */
+  sqlite3_value **apVal,          /* Array of arguments */
+  sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
+){
+  return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid);
 }
 
-/* Initialize pReader with the given root data (if iStartBlockid==0
-** the leaf data was entirely contained in the root), or from the
-** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
+/*
+** Implementation of xSync() method. Flush the contents of the pending-terms
+** hash-table to the database.
 */
-static int leavesReaderInit(fulltext_vtab *v,
-                            int idx,
-                            sqlite_int64 iStartBlockid,
-                            sqlite_int64 iEndBlockid,
-                            const char *pRootData, int nRootData,
-                            LeavesReader *pReader){
-  CLEAR(pReader);
-  pReader->idx = idx;
-
-  dataBufferInit(&pReader->rootData, 0);
-  if( iStartBlockid==0 ){
-    int rc;
-    /* Corrupt if this can't be a leaf node. */
-    if( pRootData==NULL || nRootData<1 || pRootData[0]!='\0' ){
-      return SQLITE_CORRUPT_BKPT;
-    }
-    /* Entire leaf level fit in root data. */
-    dataBufferReplace(&pReader->rootData, pRootData, nRootData);
-    rc = leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
-                        &pReader->leafReader);
-    if( rc!=SQLITE_OK ){
-      dataBufferDestroy(&pReader->rootData);
-      return rc;
-    }
-  }else{
-    sqlite3_stmt *s;
-    int rc = sql_get_leaf_statement(v, idx, &s);
-    if( rc!=SQLITE_OK ) return rc;
-
-    rc = sqlite3_bind_int64(s, 1, iStartBlockid);
-    if( rc!=SQLITE_OK ) goto err;
-
-    rc = sqlite3_bind_int64(s, 2, iEndBlockid);
-    if( rc!=SQLITE_OK ) goto err;
-
-    rc = sqlite3_step(s);
-
-    /* Corrupt if interior node referenced missing leaf node. */
-    if( rc==SQLITE_DONE ){
-      rc = SQLITE_CORRUPT_BKPT;
-      goto err;
-    }
-
-    if( rc!=SQLITE_ROW ) goto err;
-    rc = SQLITE_OK;
-
-    /* Corrupt if leaf data isn't a blob. */
-    if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
-      rc = SQLITE_CORRUPT_BKPT;
-    }else{
-      const char *pLeafData = sqlite3_column_blob(s, 0);
-      int nLeafData = sqlite3_column_bytes(s, 0);
-
-      /* Corrupt if this can't be a leaf node. */
-      if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
-        rc = SQLITE_CORRUPT_BKPT;
-      }else{
-        rc = leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
-      }
-    }
-
- err:
-    if( rc!=SQLITE_OK ){
-      if( idx==-1 ){
-        sqlite3_finalize(s);
-      }else{
-        sqlite3_reset(s);
-      }
-      return rc;
-    }
-
-    pReader->pStmt = s;
-  }
-  return SQLITE_OK;
+static int fts3SyncMethod(sqlite3_vtab *pVtab){
+  return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
 }
 
-/* Step the current leaf forward to the next term.  If we reach the
-** end of the current leaf, step forward to the next leaf block.
+/*
+** Implementation of xBegin() method. This is a no-op.
 */
-static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
-  int rc;
-  assert( !leavesReaderAtEnd(pReader) );
-  rc = leafReaderStep(&pReader->leafReader);
-  if( rc!=SQLITE_OK ) return rc;
-
-  if( leafReaderAtEnd(&pReader->leafReader) ){
-    if( pReader->rootData.pData ){
-      pReader->eof = 1;
-      return SQLITE_OK;
-    }
-    rc = sqlite3_step(pReader->pStmt);
-    if( rc!=SQLITE_ROW ){
-      pReader->eof = 1;
-      return rc==SQLITE_DONE ? SQLITE_OK : rc;
-    }
-
-    /* Corrupt if leaf data isn't a blob. */
-    if( sqlite3_column_type(pReader->pStmt, 0)!=SQLITE_BLOB ){
-      return SQLITE_CORRUPT_BKPT;
-    }else{
-      LeafReader tmp;
-      const char *pLeafData = sqlite3_column_blob(pReader->pStmt, 0);
-      int nLeafData = sqlite3_column_bytes(pReader->pStmt, 0);
-
-      /* Corrupt if this can't be a leaf node. */
-      if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
-        return SQLITE_CORRUPT_BKPT;
-      }
-
-      rc = leafReaderInit(pLeafData, nLeafData, &tmp);
-      if( rc!=SQLITE_OK ) return rc;
-      leafReaderDestroy(&pReader->leafReader);
-      pReader->leafReader = tmp;
-    }
-  }
+static int fts3BeginMethod(sqlite3_vtab *pVtab){
+  UNUSED_PARAMETER(pVtab);
+  assert( ((Fts3Table *)pVtab)->nPendingData==0 );
   return SQLITE_OK;
 }
 
-/* Order LeavesReaders by their term, ignoring idx.  Readers at eof
-** always sort to the end.
-*/
-static int leavesReaderTermCmp(LeavesReader *lr1, LeavesReader *lr2){
-  if( leavesReaderAtEnd(lr1) ){
-    if( leavesReaderAtEnd(lr2) ) return 0;
-    return 1;
-  }
-  if( leavesReaderAtEnd(lr2) ) return -1;
-
-  return leafReaderTermCmp(&lr1->leafReader,
-                           leavesReaderTerm(lr2), leavesReaderTermBytes(lr2),
-                           0);
-}
-
-/* Similar to leavesReaderTermCmp(), with additional ordering by idx
-** so that older segments sort before newer segments.
-*/
-static int leavesReaderCmp(LeavesReader *lr1, LeavesReader *lr2){
-  int c = leavesReaderTermCmp(lr1, lr2);
-  if( c!=0 ) return c;
-  return lr1->idx-lr2->idx;
-}
-
-/* Assume that pLr[1]..pLr[nLr] are sorted.  Bubble pLr[0] into its
-** sorted position.
-*/
-static void leavesReaderReorder(LeavesReader *pLr, int nLr){
-  while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){
-    LeavesReader tmp = pLr[0];
-    pLr[0] = pLr[1];
-    pLr[1] = tmp;
-    nLr--;
-    pLr++;
-  }
-}
-
-/* Initializes pReaders with the segments from level iLevel, returning
-** the number of segments in *piReaders.  Leaves pReaders in sorted
-** order.
+/*
+** Implementation of xCommit() method. This is a no-op. The contents of
+** the pending-terms hash-table have already been flushed into the database
+** by fts3SyncMethod().
 */
-static int leavesReadersInit(fulltext_vtab *v, int iLevel,
-                             LeavesReader *pReaders, int *piReaders){
-  sqlite3_stmt *s;
-  int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int(s, 1, iLevel);
-  if( rc!=SQLITE_OK ) return rc;
-
-  i = 0;
-  while( (rc = sqlite3_step(s))==SQLITE_ROW ){
-    sqlite_int64 iStart = sqlite3_column_int64(s, 0);
-    sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
-    const char *pRootData = sqlite3_column_blob(s, 2);
-    int nRootData = sqlite3_column_bytes(s, 2);
-    sqlite_int64 iIndex = sqlite3_column_int64(s, 3);
-
-    /* Corrupt if we get back different types than we stored. */
-    /* Also corrupt if the index is not sequential starting at 0. */
-    if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
-        sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-        sqlite3_column_type(s, 2)!=SQLITE_BLOB ||
-        i!=iIndex ||
-        i>=MERGE_COUNT ){
-      rc = SQLITE_CORRUPT_BKPT;
-      break;
-    }
-
-    rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
-                          &pReaders[i]);
-    if( rc!=SQLITE_OK ) break;
-
-    i++;
-  }
-  if( rc!=SQLITE_DONE ){
-    while( i-->0 ){
-      leavesReaderDestroy(&pReaders[i]);
-    }
-    sqlite3_reset(s);  /* So we don't leave a lock. */
-    return rc;
-  }
-
-  *piReaders = i;
-
-  /* Leave our results sorted by term, then age. */
-  while( i-- ){
-    leavesReaderReorder(pReaders+i, *piReaders-i);
-  }
+static int fts3CommitMethod(sqlite3_vtab *pVtab){
+  UNUSED_PARAMETER(pVtab);
+  assert( ((Fts3Table *)pVtab)->nPendingData==0 );
   return SQLITE_OK;
 }
 
-/* Merge doclists from pReaders[nReaders] into a single doclist, which
-** is written to pWriter.  Assumes pReaders is ordered oldest to
-** newest.
-*/
-/* TODO(shess) Consider putting this inline in segmentMerge(). */
-static int leavesReadersMerge(fulltext_vtab *v,
-                              LeavesReader *pReaders, int nReaders,
-                              LeafWriter *pWriter){
-  DLReader dlReaders[MERGE_COUNT];
-  const char *pTerm = leavesReaderTerm(pReaders);
-  int i, nTerm = leavesReaderTermBytes(pReaders);
-  int rc;
-
-  assert( nReaders<=MERGE_COUNT );
-
-  for(i=0; i<nReaders; i++){
-    const char *pData = leavesReaderData(pReaders+i);
-    if( pData==NULL ){
-      rc = SQLITE_CORRUPT_BKPT;
-      break;
-    }
-    rc = dlrInit(&dlReaders[i], DL_DEFAULT,
-                 pData,
-                 leavesReaderDataBytes(pReaders+i));
-    if( rc!=SQLITE_OK ) break;
-  }
-  if( rc!=SQLITE_OK ){
-    while( i-->0 ){
-      dlrDestroy(&dlReaders[i]);
-    }
-    return rc;
-  }
-
-  return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
-}
-
-/* Forward ref due to mutual recursion with segdirNextIndex(). */
-static int segmentMerge(fulltext_vtab *v, int iLevel);
-
-/* Put the next available index at iLevel into *pidx.  If iLevel
-** already has MERGE_COUNT segments, they are merged to a higher
-** level to make room.
+/*
+** Implementation of xRollback(). Discard the contents of the pending-terms
+** hash-table. Any changes made to the database are reverted by SQLite.
 */
-static int segdirNextIndex(fulltext_vtab *v, int iLevel, int *pidx){
-  int rc = segdir_max_index(v, iLevel, pidx);
-  if( rc==SQLITE_DONE ){              /* No segments at iLevel. */
-    *pidx = 0;
-  }else if( rc==SQLITE_ROW ){
-    if( *pidx==(MERGE_COUNT-1) ){
-      rc = segmentMerge(v, iLevel);
-      if( rc!=SQLITE_OK ) return rc;
-      *pidx = 0;
-    }else{
-      (*pidx)++;
-    }
-  }else{
-    return rc;
-  }
+static int fts3RollbackMethod(sqlite3_vtab *pVtab){
+  sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab);
   return SQLITE_OK;
 }
 
-/* Merge MERGE_COUNT segments at iLevel into a new segment at
-** iLevel+1.  If iLevel+1 is already full of segments, those will be
-** merged to make room.
+/*
+** Load the doclist associated with expression pExpr to pExpr->aDoclist.
+** The loaded doclist contains positions as well as the document ids.
+** This is used by the matchinfo(), snippet() and offsets() auxillary
+** functions.
 */
-static int segmentMerge(fulltext_vtab *v, int iLevel){
-  LeafWriter writer;
-  LeavesReader lrs[MERGE_COUNT];
-  int i, rc, idx = 0;
-
-  /* Determine the next available segment index at the next level,
-  ** merging as necessary.
-  */
-  rc = segdirNextIndex(v, iLevel+1, &idx);
-  if( rc!=SQLITE_OK ) return rc;
-
-  /* TODO(shess) This assumes that we'll always see exactly
-  ** MERGE_COUNT segments to merge at a given level.  That will be
-  ** broken if we allow the developer to request preemptive or
-  ** deferred merging.
-  */
-  memset(&lrs, '\0', sizeof(lrs));
-  rc = leavesReadersInit(v, iLevel, lrs, &i);
-  if( rc!=SQLITE_OK ) return rc;
-
-  leafWriterInit(iLevel+1, idx, &writer);
-
-  if( i!=MERGE_COUNT ){
-    rc = SQLITE_CORRUPT_BKPT;
-    goto err;
-  }
-
-  /* Since leavesReaderReorder() pushes readers at eof to the end,
-  ** when the first reader is empty, all will be empty.
-  */
-  while( !leavesReaderAtEnd(lrs) ){
-    /* Figure out how many readers share their next term. */
-    for(i=1; i<MERGE_COUNT && !leavesReaderAtEnd(lrs+i); i++){
-      if( 0!=leavesReaderTermCmp(lrs, lrs+i) ) break;
-    }
-
-    rc = leavesReadersMerge(v, lrs, i, &writer);
-    if( rc!=SQLITE_OK ) goto err;
-
-    /* Step forward those that were merged. */
-    while( i-->0 ){
-      rc = leavesReaderStep(v, lrs+i);
-      if( rc!=SQLITE_OK ) goto err;
-
-      /* Reorder by term, then by age. */
-      leavesReaderReorder(lrs+i, MERGE_COUNT-i);
-    }
-  }
-
-  for(i=0; i<MERGE_COUNT; i++){
-    leavesReaderDestroy(&lrs[i]);
-  }
-
-  rc = leafWriterFinalize(v, &writer);
-  leafWriterDestroy(&writer);
-  if( rc!=SQLITE_OK ) return rc;
-
-  /* Delete the merged segment data. */
-  return segdir_delete(v, iLevel);
-
- err:
-  for(i=0; i<MERGE_COUNT; i++){
-    leavesReaderDestroy(&lrs[i]);
-  }
-  leafWriterDestroy(&writer);
-  return rc;
-}
-
-/* Accumulate the union of *acc and *pData into *acc. */
-static int docListAccumulateUnion(DataBuffer *acc,
-                                  const char *pData, int nData) {
-  DataBuffer tmp = *acc;
-  int rc;
-  dataBufferInit(acc, tmp.nData+nData);
-  rc = docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
-  dataBufferDestroy(&tmp);
-  return rc;
+int sqlite3Fts3ExprLoadDoclist(Fts3Table *pTab, Fts3Expr *pExpr){
+  return evalFts3Expr(pTab, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
 }
 
-/* TODO(shess) It might be interesting to explore different merge
-** strategies, here.  For instance, since this is a sorted merge, we
-** could easily merge many doclists in parallel.  With some
-** comprehension of the storage format, we could merge all of the
-** doclists within a leaf node directly from the leaf node's storage.
-** It may be worthwhile to merge smaller doclists before larger
-** doclists, since they can be traversed more quickly - but the
-** results may have less overlap, making them more expensive in a
-** different way.
-*/
-
-/* Scan pReader for pTerm/nTerm, and merge the term's doclist over
-** *out (any doclists with duplicate docids overwrite those in *out).
-** Internal function for loadSegmentLeaf().
-*/
-static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader,
-                                const char *pTerm, int nTerm, int isPrefix,
-                                DataBuffer *out){
-  /* doclist data is accumulated into pBuffers similar to how one does
-  ** increment in binary arithmetic.  If index 0 is empty, the data is
-  ** stored there.  If there is data there, it is merged and the
-  ** results carried into position 1, with further merge-and-carry
-  ** until an empty position is found.
-  */
-  DataBuffer *pBuffers = NULL;
-  int nBuffers = 0, nMaxBuffers = 0, rc;
-
-  assert( nTerm>0 );
-
-  for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader);
-      rc=leavesReaderStep(v, pReader)){
-    /* TODO(shess) Really want leavesReaderTermCmp(), but that name is
-    ** already taken to compare the terms of two LeavesReaders.  Think
-    ** on a better name.  [Meanwhile, break encapsulation rather than
-    ** use a confusing name.]
-    */
-    int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
-    if( c>0 ) break;      /* Past any possible matches. */
-    if( c==0 ){
-      int iBuffer, nData;
-      const char *pData = leavesReaderData(pReader);
-      if( pData==NULL ){
-        rc = SQLITE_CORRUPT_BKPT;
-        break;
-      }
-      nData = leavesReaderDataBytes(pReader);
-
-      /* Find the first empty buffer. */
-      for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
-        if( 0==pBuffers[iBuffer].nData ) break;
-      }
-
-      /* Out of buffers, add an empty one. */
-      if( iBuffer==nBuffers ){
-        if( nBuffers==nMaxBuffers ){
-          DataBuffer *p;
-          nMaxBuffers += 20;
-
-          /* Manual realloc so we can handle NULL appropriately. */
-          p = sqlite3_malloc(nMaxBuffers*sizeof(*pBuffers));
-          if( p==NULL ){
-            rc = SQLITE_NOMEM;
-            break;
-          }
-
-          if( nBuffers>0 ){
-            assert(pBuffers!=NULL);
-            memcpy(p, pBuffers, nBuffers*sizeof(*pBuffers));
-            sqlite3_free(pBuffers);
-          }
-          pBuffers = p;
+/*
+** After ExprLoadDoclist() (see above) has been called, this function is
+** used to iterate/search through the position lists that make up the doclist
+** stored in pExpr->aDoclist.
+*/
+char *sqlite3Fts3FindPositions(
+  Fts3Expr *pExpr,                /* Access this expressions doclist */
+  sqlite3_int64 iDocid,           /* Docid associated with requested pos-list */
+  int iCol                        /* Column of requested pos-list */
+){
+  assert( pExpr->isLoaded );
+  if( pExpr->aDoclist ){
+    char *pEnd = &pExpr->aDoclist[pExpr->nDoclist];
+    char *pCsr = pExpr->pCurrent;
+
+    assert( pCsr );
+    while( pCsr<pEnd ){
+      if( pExpr->iCurrent<iDocid ){
+        fts3PoslistCopy(0, &pCsr);
+        if( pCsr<pEnd ){
+          fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
         }
-        dataBufferInit(&(pBuffers[nBuffers]), 0);
-        nBuffers++;
-      }
-
-      /* At this point, must have an empty at iBuffer. */
-      assert(iBuffer<nBuffers && pBuffers[iBuffer].nData==0);
-
-      /* If empty was first buffer, no need for merge logic. */
-      if( iBuffer==0 ){
-        dataBufferReplace(&(pBuffers[0]), pData, nData);
+        pExpr->pCurrent = pCsr;
       }else{
-        /* pAcc is the empty buffer the merged data will end up in. */
-        DataBuffer *pAcc = &(pBuffers[iBuffer]);
-        DataBuffer *p = &(pBuffers[0]);
-
-        /* Handle position 0 specially to avoid need to prime pAcc
-        ** with pData/nData.
-        */
-        dataBufferSwap(p, pAcc);
-        rc = docListAccumulateUnion(pAcc, pData, nData);
-        if( rc!=SQLITE_OK ) goto err;
-
-        /* Accumulate remaining doclists into pAcc. */
-        for(++p; p<pAcc; ++p){
-          rc = docListAccumulateUnion(pAcc, p->pData, p->nData);
-          if( rc!=SQLITE_OK ) goto err;
-
-          /* dataBufferReset() could allow a large doclist to blow up
-          ** our memory requirements.
-          */
-          if( p->nCapacity<1024 ){
-            dataBufferReset(p);
-          }else{
-            dataBufferDestroy(p);
-            dataBufferInit(p, 0);
+        if( pExpr->iCurrent==iDocid ){
+          int iThis = 0;
+          if( iCol<0 ){
+            /* If iCol is negative, return a pointer to the start of the
+            ** position-list (instead of a pointer to the start of a list
+            ** of offsets associated with a specific column).
+            */
+            return pCsr;
           }
+          while( iThis<iCol ){
+            fts3ColumnlistCopy(0, &pCsr);
+            if( *pCsr==0x00 ) return 0;
+            pCsr++;
+            pCsr += sqlite3Fts3GetVarint32(pCsr, &iThis);
+          }
+          if( iCol==iThis && (*pCsr&0xFE) ) return pCsr;
         }
+        return 0;
       }
     }
   }
 
-  /* Union all the doclists together into *out. */
-  /* TODO(shess) What if *out is big?  Sigh. */
-  if( rc==SQLITE_OK && nBuffers>0 ){
-    int iBuffer;
-    for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
-      if( pBuffers[iBuffer].nData>0 ){
-        if( out->nData==0 ){
-          dataBufferSwap(out, &(pBuffers[iBuffer]));
-        }else{
-          rc = docListAccumulateUnion(out, pBuffers[iBuffer].pData,
-                                      pBuffers[iBuffer].nData);
-          if( rc!=SQLITE_OK ) break;
-        }
-      }
-    }
-  }
-
-err:
-  while( nBuffers-- ){
-    dataBufferDestroy(&(pBuffers[nBuffers]));
-  }
-  if( pBuffers!=NULL ) sqlite3_free(pBuffers);
-
-  return rc;
-}
-
-/* Call loadSegmentLeavesInt() with pData/nData as input. */
-static int loadSegmentLeaf(fulltext_vtab *v, const char *pData, int nData,
-                           const char *pTerm, int nTerm, int isPrefix,
-                           DataBuffer *out){
-  LeavesReader reader;
-  int rc;
-
-  assert( nData>1 );
-  assert( *pData=='\0' );
-  rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
-  leavesReaderReset(&reader);
-  leavesReaderDestroy(&reader);
-  return rc;
-}
-
-/* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to
-** iEndLeaf (inclusive) as input, and merge the resulting doclist into
-** out.
-*/
-static int loadSegmentLeaves(fulltext_vtab *v,
-                             sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf,
-                             const char *pTerm, int nTerm, int isPrefix,
-                             DataBuffer *out){
-  int rc;
-  LeavesReader reader;
-
-  assert( iStartLeaf<=iEndLeaf );
-  rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
-  leavesReaderReset(&reader);
-  leavesReaderDestroy(&reader);
-  return rc;
-}
-
-/* Taking pData/nData as an interior node, find the sequence of child
-** nodes which could include pTerm/nTerm/isPrefix.  Note that the
-** interior node terms logically come between the blocks, so there is
-** one more blockid than there are terms (that block contains terms >=
-** the last interior-node term).
-*/
-/* TODO(shess) The calling code may already know that the end child is
-** not worth calculating, because the end may be in a later sibling
-** node.  Consider whether breaking symmetry is worthwhile.  I suspect
-** it is not worthwhile.
-*/
-static int getChildrenContaining(const char *pData, int nData,
-                                 const char *pTerm, int nTerm, int isPrefix,
-                                 sqlite_int64 *piStartChild,
-                                 sqlite_int64 *piEndChild){
-  InteriorReader reader;
-  int rc;
-
-  assert( nData>1 );
-  assert( *pData!='\0' );
-  rc = interiorReaderInit(pData, nData, &reader);
-  if( rc!=SQLITE_OK ) return rc;
-
-  /* Scan for the first child which could contain pTerm/nTerm. */
-  while( !interiorReaderAtEnd(&reader) ){
-    if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
-    rc = interiorReaderStep(&reader);
-    if( rc!=SQLITE_OK ){
-      interiorReaderDestroy(&reader);
-      return rc;
-    }
-  }
-  *piStartChild = interiorReaderCurrentBlockid(&reader);
-
-  /* Keep scanning to find a term greater than our term, using prefix
-  ** comparison if indicated.  If isPrefix is false, this will be the
-  ** same blockid as the starting block.
-  */
-  while( !interiorReaderAtEnd(&reader) ){
-    if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
-    rc = interiorReaderStep(&reader);
-    if( rc!=SQLITE_OK ){
-      interiorReaderDestroy(&reader);
-      return rc;
-    }
-  }
-  *piEndChild = interiorReaderCurrentBlockid(&reader);
-
-  interiorReaderDestroy(&reader);
-
-  /* Children must ascend, and if !prefix, both must be the same. */
-  assert( *piEndChild>=*piStartChild );
-  assert( isPrefix || *piStartChild==*piEndChild );
-  return rc;
-}
-
-/* Read block at iBlockid and pass it with other params to
-** getChildrenContaining().
-*/
-static int loadAndGetChildrenContaining(
-  fulltext_vtab *v,
-  sqlite_int64 iBlockid,
-  const char *pTerm, int nTerm, int isPrefix,
-  sqlite_int64 *piStartChild, sqlite_int64 *piEndChild
-){
-  sqlite3_stmt *s = NULL;
-  int rc;
-
-  assert( iBlockid!=0 );
-  assert( pTerm!=NULL );
-  assert( nTerm!=0 );        /* TODO(shess) Why not allow this? */
-  assert( piStartChild!=NULL );
-  assert( piEndChild!=NULL );
-
-  rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_bind_int64(s, 1, iBlockid);
-  if( rc!=SQLITE_OK ) return rc;
-
-  rc = sqlite3_step(s);
-  /* Corrupt if interior node references missing child node. */
-  if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
-  if( rc!=SQLITE_ROW ) return rc;
-
-  /* Corrupt if child node isn't a blob. */
-  if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
-    sqlite3_reset(s);  /* So we don't leave a lock. */
-    return SQLITE_CORRUPT_BKPT;
-  }else{
-    const char *pData = sqlite3_column_blob(s, 0);
-    int nData = sqlite3_column_bytes(s, 0);
-
-    /* Corrupt if child is not a valid interior node. */
-    if( pData==NULL || nData<1 || pData[0]=='\0' ){
-      sqlite3_reset(s);  /* So we don't leave a lock. */
-      return SQLITE_CORRUPT_BKPT;
-    }
-
-    rc = getChildrenContaining(pData, nData, pTerm, nTerm,
-                               isPrefix, piStartChild, piEndChild);
-    if( rc!=SQLITE_OK ){
-      sqlite3_reset(s);
-      return rc;
-    }
-  }
-
-  /* We expect only one row.  We must execute another sqlite3_step()
-   * to complete the iteration; otherwise the table will remain
-   * locked. */
-  rc = sqlite3_step(s);
-  if( rc==SQLITE_ROW ) return SQLITE_ERROR;
-  if( rc!=SQLITE_DONE ) return rc;
-
-  return SQLITE_OK;
-}
-
-/* Traverse the tree represented by pData[nData] looking for
-** pTerm[nTerm], placing its doclist into *out.  This is internal to
-** loadSegment() to make error-handling cleaner.
-*/
-static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData,
-                          sqlite_int64 iLeavesEnd,
-                          const char *pTerm, int nTerm, int isPrefix,
-                          DataBuffer *out){
-  /* Special case where root is a leaf. */
-  if( *pData=='\0' ){
-    return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out);
-  }else{
-    int rc;
-    sqlite_int64 iStartChild, iEndChild;
-
-    /* Process pData as an interior node, then loop down the tree
-    ** until we find the set of leaf nodes to scan for the term.
-    */
-    rc = getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
-                               &iStartChild, &iEndChild);
-    if( rc!=SQLITE_OK ) return rc;
-    while( iStartChild>iLeavesEnd ){
-      sqlite_int64 iNextStart, iNextEnd;
-      rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
-                                        &iNextStart, &iNextEnd);
-      if( rc!=SQLITE_OK ) return rc;
-
-      /* If we've branched, follow the end branch, too. */
-      if( iStartChild!=iEndChild ){
-        sqlite_int64 iDummy;
-        rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix,
-                                          &iDummy, &iNextEnd);
-        if( rc!=SQLITE_OK ) return rc;
-      }
-
-      assert( iNextStart<=iNextEnd );
-      iStartChild = iNextStart;
-      iEndChild = iNextEnd;
-    }
-    assert( iStartChild<=iLeavesEnd );
-    assert( iEndChild<=iLeavesEnd );
-
-    /* Scan through the leaf segments for doclists. */
-    return loadSegmentLeaves(v, iStartChild, iEndChild,
-                             pTerm, nTerm, isPrefix, out);
-  }
-}
-
-/* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then
-** merge its doclist over *out (any duplicate doclists read from the
-** segment rooted at pData will overwrite those in *out).
-*/
-/* TODO(shess) Consider changing this to determine the depth of the
-** leaves using either the first characters of interior nodes (when
-** ==1, we're one level above the leaves), or the first character of
-** the root (which will describe the height of the tree directly).
-** Either feels somewhat tricky to me.
-*/
-/* TODO(shess) The current merge is likely to be slow for large
-** doclists (though it should process from newest/smallest to
-** oldest/largest, so it may not be that bad).  It might be useful to
-** modify things to allow for N-way merging.  This could either be
-** within a segment, with pairwise merges across segments, or across
-** all segments at once.
-*/
-static int loadSegment(fulltext_vtab *v, const char *pData, int nData,
-                       sqlite_int64 iLeavesEnd,
-                       const char *pTerm, int nTerm, int isPrefix,
-                       DataBuffer *out){
-  DataBuffer result;
-  int rc;
-
-  /* Corrupt if segment root can't be valid. */
-  if( pData==NULL || nData<1 ) return SQLITE_CORRUPT_BKPT;
-
-  /* This code should never be called with buffered updates. */
-  assert( v->nPendingData<0 );
-
-  dataBufferInit(&result, 0);
-  rc = loadSegmentInt(v, pData, nData, iLeavesEnd,
-                      pTerm, nTerm, isPrefix, &result);
-  if( rc==SQLITE_OK && result.nData>0 ){
-    if( out->nData==0 ){
-      DataBuffer tmp = *out;
-      *out = result;
-      result = tmp;
-    }else{
-      DataBuffer merged;
-      DLReader readers[2];
-
-      rc = dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
-      if( rc==SQLITE_OK ){
-        rc = dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
-        if( rc==SQLITE_OK ){
-          dataBufferInit(&merged, out->nData+result.nData);
-          rc = docListMerge(&merged, readers, 2);
-          dataBufferDestroy(out);
-          *out = merged;
-          dlrDestroy(&readers[1]);
-        }
-        dlrDestroy(&readers[0]);
-      }
-    }
-  }
-
-  dataBufferDestroy(&result);
-  return rc;
+  return 0;
 }
 
-/* Scan the database and merge together the posting lists for the term
-** into *out.
-*/
-static int termSelect(
-  fulltext_vtab *v, 
-  int iColumn,
-  const char *pTerm, int nTerm,             /* Term to query for */
-  int isPrefix,                             /* True for a prefix search */
-  DocListType iType, 
-  DataBuffer *out                           /* Write results here */
+/*
+** Helper function used by the implementation of the overloaded snippet(),
+** offsets() and optimize() SQL functions.
+**
+** If the value passed as the third argument is a blob of size
+** sizeof(Fts3Cursor*), then the blob contents are copied to the 
+** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error
+** message is written to context pContext and SQLITE_ERROR returned. The
+** string passed via zFunc is used as part of the error message.
+*/
+static int fts3FunctionArg(
+  sqlite3_context *pContext,      /* SQL function call context */
+  const char *zFunc,              /* Function name */
+  sqlite3_value *pVal,            /* argv[0] passed to function */
+  Fts3Cursor **ppCsr         /* OUT: Store cursor handle here */
 ){
-  DataBuffer doclist;
-  sqlite3_stmt *s;
-  int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
-  if( rc!=SQLITE_OK ) return rc;
-
-  /* This code should never be called with buffered updates. */
-  assert( v->nPendingData<0 );
-
-  dataBufferInit(&doclist, 0);
-  dataBufferInit(out, 0);
-
-  /* Traverse the segments from oldest to newest so that newer doclist
-  ** elements for given docids overwrite older elements.
-  */
-  while( (rc = sqlite3_step(s))==SQLITE_ROW ){
-    const char *pData = sqlite3_column_blob(s, 2);
-    const int nData = sqlite3_column_bytes(s, 2);
-    const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
-
-    /* Corrupt if we get back different types than we stored. */
-    if( sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-        sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
-      rc = SQLITE_CORRUPT_BKPT;
-      goto err;
-    }
-
-    rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
-                     &doclist);
-    if( rc!=SQLITE_OK ) goto err;
-  }
-  if( rc==SQLITE_DONE ){
-    rc = SQLITE_OK;
-    if( doclist.nData!=0 ){
-      /* TODO(shess) The old term_select_all() code applied the column
-      ** restrict as we merged segments, leading to smaller buffers.
-      ** This is probably worthwhile to bring back, once the new storage
-      ** system is checked in.
-      */
-      if( iColumn==v->nColumn) iColumn = -1;
-      rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
-                       iColumn, iType, out);
-    }
-  }
-
- err:
-  sqlite3_reset(s);  /* So we don't leave a lock. */
-  dataBufferDestroy(&doclist);
-  return rc;
-}
-
-/****************************************************************/
-/* Used to hold hashtable data for sorting. */
-typedef struct TermData {
-  const char *pTerm;
-  int nTerm;
-  DLCollector *pCollector;
-} TermData;
-
-/* Orders TermData elements in strcmp fashion ( <0 for less-than, 0
-** for equal, >0 for greater-than).
-*/
-static int termDataCmp(const void *av, const void *bv){
-  const TermData *a = (const TermData *)av;
-  const TermData *b = (const TermData *)bv;
-  int n = a->nTerm<b->nTerm ? a->nTerm : b->nTerm;
-  int c = memcmp(a->pTerm, b->pTerm, n);
-  if( c!=0 ) return c;
-  return a->nTerm-b->nTerm;
-}
-
-/* Order pTerms data by term, then write a new level 0 segment using
-** LeafWriter.
-*/
-static int writeZeroSegment(fulltext_vtab *v, fts3Hash *pTerms){
-  fts3HashElem *e;
-  int idx, rc, i, n;
-  TermData *pData;
-  LeafWriter writer;
-  DataBuffer dl;
-
-  /* Determine the next index at level 0, merging as necessary. */
-  rc = segdirNextIndex(v, 0, &idx);
-  if( rc!=SQLITE_OK ) return rc;
-
-  n = fts3HashCount(pTerms);
-  pData = sqlite3_malloc(n*sizeof(TermData));
-
-  for(i = 0, e = fts3HashFirst(pTerms); e; i++, e = fts3HashNext(e)){
-    assert( i<n );
-    pData[i].pTerm = fts3HashKey(e);
-    pData[i].nTerm = fts3HashKeysize(e);
-    pData[i].pCollector = fts3HashData(e);
-  }
-  assert( i==n );
-
-  /* TODO(shess) Should we allow user-defined collation sequences,
-  ** here?  I think we only need that once we support prefix searches.
-  */
-  if( n>1 ) qsort(pData, n, sizeof(*pData), termDataCmp);
-
-  /* TODO(shess) Refactor so that we can write directly to the segment
-  ** DataBuffer, as happens for segment merges.
-  */
-  leafWriterInit(0, idx, &writer);
-  dataBufferInit(&dl, 0);
-  for(i=0; i<n; i++){
-    dataBufferReset(&dl);
-    dlcAddDoclist(pData[i].pCollector, &dl);
-    rc = leafWriterStep(v, &writer,
-                        pData[i].pTerm, pData[i].nTerm, dl.pData, dl.nData);
-    if( rc!=SQLITE_OK ) goto err;
-  }
-  rc = leafWriterFinalize(v, &writer);
-
- err:
-  dataBufferDestroy(&dl);
-  sqlite3_free(pData);
-  leafWriterDestroy(&writer);
-  return rc;
-}
-
-/* If pendingTerms has data, free it. */
-static int clearPendingTerms(fulltext_vtab *v){
-  if( v->nPendingData>=0 ){
-    fts3HashElem *e;
-    for(e=fts3HashFirst(&v->pendingTerms); e; e=fts3HashNext(e)){
-      dlcDelete(fts3HashData(e));
-    }
-    fts3HashClear(&v->pendingTerms);
-    v->nPendingData = -1;
-  }
-  return SQLITE_OK;
-}
-
-/* If pendingTerms has data, flush it to a level-zero segment, and
-** free it.
-*/
-static int flushPendingTerms(fulltext_vtab *v){
-  if( v->nPendingData>=0 ){
-    int rc = writeZeroSegment(v, &v->pendingTerms);
-    if( rc==SQLITE_OK ) clearPendingTerms(v);
-    return rc;
-  }
-  return SQLITE_OK;
-}
-
-/* If pendingTerms is "too big", or docid is out of order, flush it.
-** Regardless, be certain that pendingTerms is initialized for use.
-*/
-static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
-  /* TODO(shess) Explore whether partially flushing the buffer on
-  ** forced-flush would provide better performance.  I suspect that if
-  ** we ordered the doclists by size and flushed the largest until the
-  ** buffer was half empty, that would let the less frequent terms
-  ** generate longer doclists.
-  */
-  if( iDocid<=v->iPrevDocid || v->nPendingData>kPendingThreshold ){
-    int rc = flushPendingTerms(v);
-    if( rc!=SQLITE_OK ) return rc;
-  }
-  if( v->nPendingData<0 ){
-    fts3HashInit(&v->pendingTerms, FTS3_HASH_STRING, 1);
-    v->nPendingData = 0;
+  Fts3Cursor *pRet;
+  if( sqlite3_value_type(pVal)!=SQLITE_BLOB 
+   || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
+  ){
+    char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
+    sqlite3_result_error(pContext, zErr, -1);
+    sqlite3_free(zErr);
+    return SQLITE_ERROR;
   }
-  v->iPrevDocid = iDocid;
+  memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
+  *ppCsr = pRet;
   return SQLITE_OK;
 }
 
-/* This function implements the xUpdate callback; it is the top-level entry
- * point for inserting, deleting or updating a row in a full-text table. */
-static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
-                          sqlite_int64 *pRowid){
-  fulltext_vtab *v = (fulltext_vtab *) pVtab;
-  int rc;
-
-  FTSTRACE(("FTS3 Update %p\n", pVtab));
-
-  if( nArg<2 ){
-    rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
-    if( rc==SQLITE_OK ){
-      /* If we just deleted the last row in the table, clear out the
-      ** index data.
-      */
-      rc = content_exists(v);
-      if( rc==SQLITE_ROW ){
-        rc = SQLITE_OK;
-      }else if( rc==SQLITE_DONE ){
-        /* Clear the pending terms so we don't flush a useless level-0
-        ** segment when the transaction closes.
-        */
-        rc = clearPendingTerms(v);
-        if( rc==SQLITE_OK ){
-          rc = segdir_delete_all(v);
-        }
-      }
-    }
-  } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
-    /* An update:
-     * ppArg[0] = old rowid
-     * ppArg[1] = new rowid
-     * ppArg[2..2+v->nColumn-1] = values
-     * ppArg[2+v->nColumn] = value for magic column (we ignore this)
-     * ppArg[2+v->nColumn+1] = value for docid
-     */
-    sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
-    if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER ||
-        sqlite3_value_int64(ppArg[1]) != rowid ){
-      rc = SQLITE_ERROR;  /* we don't allow changing the rowid */
-    }else if( sqlite3_value_type(ppArg[2+v->nColumn+1]) != SQLITE_INTEGER ||
-              sqlite3_value_int64(ppArg[2+v->nColumn+1]) != rowid ){
-      rc = SQLITE_ERROR;  /* we don't allow changing the docid */
-    }else{
-      assert( nArg==2+v->nColumn+2);
-      rc = index_update(v, rowid, &ppArg[2]);
-    }
-  } else {
-    /* An insert:
-     * ppArg[1] = requested rowid
-     * ppArg[2..2+v->nColumn-1] = values
-     * ppArg[2+v->nColumn] = value for magic column (we ignore this)
-     * ppArg[2+v->nColumn+1] = value for docid
-     */
-    sqlite3_value *pRequestDocid = ppArg[2+v->nColumn+1];
-    assert( nArg==2+v->nColumn+2);
-    if( SQLITE_NULL != sqlite3_value_type(pRequestDocid) &&
-        SQLITE_NULL != sqlite3_value_type(ppArg[1]) ){
-      /* TODO(shess) Consider allowing this to work if the values are
-      ** identical.  I'm inclined to discourage that usage, though,
-      ** given that both rowid and docid are special columns.  Better
-      ** would be to define one or the other as the default winner,
-      ** but should it be fts3-centric (docid) or SQLite-centric
-      ** (rowid)?
-      */
-      rc = SQLITE_ERROR;
-    }else{
-      if( SQLITE_NULL == sqlite3_value_type(pRequestDocid) ){
-        pRequestDocid = ppArg[1];
-      }
-      rc = index_insert(v, pRequestDocid, &ppArg[2], pRowid);
-    }
-  }
-
-  return rc;
-}
-
-static int fulltextSync(sqlite3_vtab *pVtab){
-  FTSTRACE(("FTS3 xSync()\n"));
-  return flushPendingTerms((fulltext_vtab *)pVtab);
-}
-
-static int fulltextBegin(sqlite3_vtab *pVtab){
-  fulltext_vtab *v = (fulltext_vtab *) pVtab;
-  FTSTRACE(("FTS3 xBegin()\n"));
-
-  /* Any buffered updates should have been cleared by the previous
-  ** transaction.
-  */
-  assert( v->nPendingData<0 );
-  return clearPendingTerms(v);
-}
-
-static int fulltextCommit(sqlite3_vtab *pVtab){
-  fulltext_vtab *v = (fulltext_vtab *) pVtab;
-  FTSTRACE(("FTS3 xCommit()\n"));
-
-  /* Buffered updates should have been cleared by fulltextSync(). */
-  assert( v->nPendingData<0 );
-  return clearPendingTerms(v);
-}
-
-static int fulltextRollback(sqlite3_vtab *pVtab){
-  FTSTRACE(("FTS3 xRollback()\n"));
-  return clearPendingTerms((fulltext_vtab *)pVtab);
-}
-
 /*
 ** Implementation of the snippet() function for FTS3
 */
-static void snippetFunc(
-  sqlite3_context *pContext,
-  int argc,
-  sqlite3_value **argv
-){
-  fulltext_cursor *pCursor;
-  if( argc<1 ) return;
-  if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
-      sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
-    sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
-  }else{
-    const char *zStart = "<b>";
-    const char *zEnd = "</b>";
-    const char *zEllipsis = "<b>...</b>";
-    memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
-    if( argc>=2 ){
-      zStart = (const char*)sqlite3_value_text(argv[1]);
-      if( argc>=3 ){
-        zEnd = (const char*)sqlite3_value_text(argv[2]);
-        if( argc>=4 ){
-          zEllipsis = (const char*)sqlite3_value_text(argv[3]);
-        }
-      }
-    }
-    snippetAllOffsets(pCursor);
-    snippetText(pCursor, zStart, zEnd, zEllipsis);
-    sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
-                        pCursor->snippet.nSnippet, SQLITE_STATIC);
-  }
-}
-
-/*
-** Implementation of the offsets() function for FTS3
-*/
-static void snippetOffsetsFunc(
-  sqlite3_context *pContext,
-  int argc,
-  sqlite3_value **argv
+static void fts3SnippetFunc(
+  sqlite3_context *pContext,      /* SQLite function call context */
+  int nVal,                       /* Size of apVal[] array */
+  sqlite3_value **apVal           /* Array of arguments */
 ){
-  fulltext_cursor *pCursor;
-  if( argc<1 ) return;
-  if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
-      sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
-    sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
-  }else{
-    memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
-    snippetAllOffsets(pCursor);
-    snippetOffsetText(&pCursor->snippet);
-    sqlite3_result_text(pContext,
-                        pCursor->snippet.zOffset, pCursor->snippet.nOffset,
-                        SQLITE_STATIC);
-  }
-}
-
-/* OptLeavesReader is nearly identical to LeavesReader, except that
-** where LeavesReader is geared towards the merging of complete
-** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader
-** is geared towards implementation of the optimize() function, and
-** can merge all segments simultaneously.  This version may be
-** somewhat less efficient than LeavesReader because it merges into an
-** accumulator rather than doing an N-way merge, but since segment
-** size grows exponentially (so segment count logrithmically) this is
-** probably not an immediate problem.
-*/
-/* TODO(shess): Prove that assertion, or extend the merge code to
-** merge tree fashion (like the prefix-searching code does).
-*/
-/* TODO(shess): OptLeavesReader and LeavesReader could probably be
-** merged with little or no loss of performance for LeavesReader.  The
-** merged code would need to handle >MERGE_COUNT segments, and would
-** also need to be able to optionally optimize away deletes.
-*/
-typedef struct OptLeavesReader {
-  /* Segment number, to order readers by age. */
-  int segment;
-  LeavesReader reader;
-} OptLeavesReader;
-
-static int optLeavesReaderAtEnd(OptLeavesReader *pReader){
-  return leavesReaderAtEnd(&pReader->reader);
-}
-static int optLeavesReaderTermBytes(OptLeavesReader *pReader){
-  return leavesReaderTermBytes(&pReader->reader);
-}
-static const char *optLeavesReaderData(OptLeavesReader *pReader){
-  return leavesReaderData(&pReader->reader);
-}
-static int optLeavesReaderDataBytes(OptLeavesReader *pReader){
-  return leavesReaderDataBytes(&pReader->reader);
-}
-static const char *optLeavesReaderTerm(OptLeavesReader *pReader){
-  return leavesReaderTerm(&pReader->reader);
-}
-static int optLeavesReaderStep(fulltext_vtab *v, OptLeavesReader *pReader){
-  return leavesReaderStep(v, &pReader->reader);
-}
-static int optLeavesReaderTermCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
-  return leavesReaderTermCmp(&lr1->reader, &lr2->reader);
-}
-/* Order by term ascending, segment ascending (oldest to newest), with
-** exhausted readers to the end.
-*/
-static int optLeavesReaderCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
-  int c = optLeavesReaderTermCmp(lr1, lr2);
-  if( c!=0 ) return c;
-  return lr1->segment-lr2->segment;
-}
-/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1].  Assumes that
-** pLr[1..nLr-1] is already sorted.
-*/
-static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){
-  while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){
-    OptLeavesReader tmp = pLr[0];
-    pLr[0] = pLr[1];
-    pLr[1] = tmp;
-    nLr--;
-    pLr++;
-  }
-}
-
-/* optimize() helper function.  Put the readers in order and iterate
-** through them, merging doclists for matching terms into pWriter.
-** Returns SQLITE_OK on success, or the SQLite error code which
-** prevented success.
-*/
-static int optimizeInternal(fulltext_vtab *v,
-                            OptLeavesReader *readers, int nReaders,
-                            LeafWriter *pWriter){
-  int i, rc = SQLITE_OK;
-  DataBuffer doclist, merged, tmp;
-  const char *pData;
-
-  /* Order the readers. */
-  i = nReaders;
-  while( i-- > 0 ){
-    optLeavesReaderReorder(&readers[i], nReaders-i);
-  }
-
-  dataBufferInit(&doclist, LEAF_MAX);
-  dataBufferInit(&merged, LEAF_MAX);
-
-  /* Exhausted readers bubble to the end, so when the first reader is
-  ** at eof, all are at eof.
+  Fts3Cursor *pCsr;               /* Cursor handle passed through apVal[0] */
+  const char *zStart = "<b>";
+  const char *zEnd = "</b>";
+  const char *zEllipsis = "<b>...</b>";
+  int iCol = -1;
+  int nToken = 15;                /* Default number of tokens in snippet */
+
+  /* There must be at least one argument passed to this function (otherwise
+  ** the non-overloaded version would have been called instead of this one).
   */
-  while( !optLeavesReaderAtEnd(&readers[0]) ){
+  assert( nVal>=1 );
 
-    /* Figure out how many readers share the next term. */
-    for(i=1; i<nReaders && !optLeavesReaderAtEnd(&readers[i]); i++){
-      if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
-    }
-
-    pData = optLeavesReaderData(&readers[0]);
-    if( pData==NULL ){
-      rc = SQLITE_CORRUPT_BKPT;
-      break;
-    }
-
-    /* Special-case for no merge. */
-    if( i==1 ){
-      /* Trim deletions from the doclist. */
-      dataBufferReset(&merged);
-      rc = docListTrim(DL_DEFAULT, pData,
-                       optLeavesReaderDataBytes(&readers[0]),
-                       -1, DL_DEFAULT, &merged);
-      if( rc!=SQLITE_OK ) break;
-    }else{
-      DLReader dlReaders[MERGE_COUNT];
-      int iReader, nReaders;
-
-      /* Prime the pipeline with the first reader's doclist.  After
-      ** one pass index 0 will reference the accumulated doclist.
-      */
-      rc = dlrInit(&dlReaders[0], DL_DEFAULT,
-                   pData,
-                   optLeavesReaderDataBytes(&readers[0]));
-      if( rc!=SQLITE_OK ) break;
-      iReader = 1;
-
-      assert( iReader<i );  /* Must execute the loop at least once. */
-      while( iReader<i ){
-        /* Merge 16 inputs per pass. */
-        for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
-             iReader++, nReaders++ ){
-          pData = optLeavesReaderData(&readers[iReader]);
-          if( pData==NULL ){
-            rc = SQLITE_CORRUPT_BKPT;
-            break;
-          }
-          rc = dlrInit(&dlReaders[nReaders], DL_DEFAULT, pData,
-                       optLeavesReaderDataBytes(&readers[iReader]));
-          if( rc!=SQLITE_OK ) break;
-        }
-
-        /* Merge doclists and swap result into accumulator. */
-        if( rc==SQLITE_OK ){
-          dataBufferReset(&merged);
-          rc = docListMerge(&merged, dlReaders, nReaders);
-          tmp = merged;
-          merged = doclist;
-          doclist = tmp;
-        }
-
-        while( nReaders-- > 0 ){
-          dlrDestroy(&dlReaders[nReaders]);
-        }
-
-        if( rc!=SQLITE_OK ) goto err;
-
-        /* Accumulated doclist to reader 0 for next pass. */
-        rc = dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
-        if( rc!=SQLITE_OK ) goto err;
-      }
-
-      /* Destroy reader that was left in the pipeline. */
-      dlrDestroy(&dlReaders[0]);
-
-      /* Trim deletions from the doclist. */
-      dataBufferReset(&merged);
-      rc = docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
-                       -1, DL_DEFAULT, &merged);
-      if( rc!=SQLITE_OK ) goto err;
-    }
-
-    /* Only pass doclists with hits (skip if all hits deleted). */
-    if( merged.nData>0 ){
-      rc = leafWriterStep(v, pWriter,
-                          optLeavesReaderTerm(&readers[0]),
-                          optLeavesReaderTermBytes(&readers[0]),
-                          merged.pData, merged.nData);
-      if( rc!=SQLITE_OK ) goto err;
-    }
-
-    /* Step merged readers to next term and reorder. */
-    while( i-- > 0 ){
-      rc = optLeavesReaderStep(v, &readers[i]);
-      if( rc!=SQLITE_OK ) goto err;
-
-      optLeavesReaderReorder(&readers[i], nReaders-i);
-    }
-  }
-
- err:
-  dataBufferDestroy(&doclist);
-  dataBufferDestroy(&merged);
-  return rc;
-}
-
-/* Implement optimize() function for FTS3.  optimize(t) merges all
-** segments in the fts index into a single segment.  't' is the magic
-** table-named column.
-*/
-static void optimizeFunc(sqlite3_context *pContext,
-                         int argc, sqlite3_value **argv){
-  fulltext_cursor *pCursor;
-  if( argc>1 ){
-    sqlite3_result_error(pContext, "excess arguments to optimize()",-1);
-  }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
-            sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
-    sqlite3_result_error(pContext, "illegal first argument to optimize",-1);
-  }else{
-    fulltext_vtab *v;
-    int i, rc, iMaxLevel;
-    OptLeavesReader *readers;
-    int nReaders;
-    LeafWriter writer;
-    sqlite3_stmt *s;
-
-    memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
-    v = cursor_vtab(pCursor);
-
-    /* Flush any buffered updates before optimizing. */
-    rc = flushPendingTerms(v);
-    if( rc!=SQLITE_OK ) goto err;
-
-    rc = segdir_count(v, &nReaders, &iMaxLevel);
-    if( rc!=SQLITE_OK ) goto err;
-    if( nReaders==0 || nReaders==1 ){
-      sqlite3_result_text(pContext, "Index already optimal", -1,
-                          SQLITE_STATIC);
-      return;
-    }
-
-    rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
-    if( rc!=SQLITE_OK ) goto err;
-
-    readers = sqlite3_malloc(nReaders*sizeof(readers[0]));
-    if( readers==NULL ) goto err;
-
-    /* Note that there will already be a segment at this position
-    ** until we call segdir_delete() on iMaxLevel.
-    */
-    leafWriterInit(iMaxLevel, 0, &writer);
-
-    i = 0;
-    while( (rc = sqlite3_step(s))==SQLITE_ROW ){
-      sqlite_int64 iStart = sqlite3_column_int64(s, 0);
-      sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
-      const char *pRootData = sqlite3_column_blob(s, 2);
-      int nRootData = sqlite3_column_bytes(s, 2);
-
-      /* Corrupt if we get back different types than we stored. */
-      if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
-          sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-          sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
-        rc = SQLITE_CORRUPT_BKPT;
-        break;
-      }
-
-      assert( i<nReaders );
-      rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
-                            &readers[i].reader);
-      if( rc!=SQLITE_OK ) break;
-
-      readers[i].segment = i;
-      i++;
-    }
-
-    /* If we managed to successfully read them all, optimize them. */
-    if( rc==SQLITE_DONE ){
-      assert( i==nReaders );
-      rc = optimizeInternal(v, readers, nReaders, &writer);
-    }else{
-      sqlite3_reset(s);  /* So we don't leave a lock. */
-    }
-
-    while( i-- > 0 ){
-      leavesReaderDestroy(&readers[i].reader);
-    }
-    sqlite3_free(readers);
-
-    /* If we've successfully gotten to here, delete the old segments
-    ** and flush the interior structure of the new segment.
-    */
-    if( rc==SQLITE_OK ){
-      for( i=0; i<=iMaxLevel; i++ ){
-        rc = segdir_delete(v, i);
-        if( rc!=SQLITE_OK ) break;
-      }
-
-      if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer);
-    }
-
-    leafWriterDestroy(&writer);
-
-    if( rc!=SQLITE_OK ) goto err;
-
-    sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
+  if( nVal>6 ){
+    sqlite3_result_error(pContext, 
+        "wrong number of arguments to function snippet()", -1);
     return;
-
-    /* TODO(shess): Error-handling needs to be improved along the
-    ** lines of the dump_ functions.
-    */
- err:
-    {
-      char buf[512];
-      sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s",
-                       sqlite3_errmsg(sqlite3_context_db_handle(pContext)));
-      sqlite3_result_error(pContext, buf, -1);
-    }
   }
-}
-
-#ifdef SQLITE_TEST
-/* Generate an error of the form "<prefix>: <msg>".  If msg is NULL,
-** pull the error from the context's db handle.
-*/
-static void generateError(sqlite3_context *pContext,
-                          const char *prefix, const char *msg){
-  char buf[512];
-  if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext));
-  sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg);
-  sqlite3_result_error(pContext, buf, -1);
-}
-
-/* Helper function to collect the set of terms in the segment into
-** pTerms.  The segment is defined by the leaf nodes between
-** iStartBlockid and iEndBlockid, inclusive, or by the contents of
-** pRootData if iStartBlockid is 0 (in which case the entire segment
-** fit in a leaf).
-*/
-static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s,
-                               fts3Hash *pTerms){
-  const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0);
-  const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
-  const char *pRootData = sqlite3_column_blob(s, 2);
-  const int nRootData = sqlite3_column_bytes(s, 2);
-  int rc;
-  LeavesReader reader;
+  if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return;
 
-  /* Corrupt if we get back different types than we stored. */
-  if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
-      sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
-      sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
-    return SQLITE_CORRUPT_BKPT;
+  switch( nVal ){
+    case 6: nToken = sqlite3_value_int(apVal[5]);
+    case 5: iCol = sqlite3_value_int(apVal[4]);
+    case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]);
+    case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]);
+    case 2: zStart = (const char*)sqlite3_value_text(apVal[1]);
   }
-
-  rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
-                        pRootData, nRootData, &reader);
-  if( rc!=SQLITE_OK ) return rc;
-
-  while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
-    const char *pTerm = leavesReaderTerm(&reader);
-    const int nTerm = leavesReaderTermBytes(&reader);
-    void *oldValue = sqlite3Fts3HashFind(pTerms, pTerm, nTerm);
-    void *newValue = (void *)((char *)oldValue+1);
-
-    /* From the comment before sqlite3Fts3HashInsert in fts3_hash.c,
-    ** the data value passed is returned in case of malloc failure.
-    */
-    if( newValue==sqlite3Fts3HashInsert(pTerms, pTerm, nTerm, newValue) ){
-      rc = SQLITE_NOMEM;
-    }else{
-      rc = leavesReaderStep(v, &reader);
-    }
+  if( !zEllipsis || !zEnd || !zStart ){
+    sqlite3_result_error_nomem(pContext);
+  }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){
+    sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken);
   }
-
-  leavesReaderDestroy(&reader);
-  return rc;
 }
 
-/* Helper function to build the result string for dump_terms(). */
-static int generateTermsResult(sqlite3_context *pContext, fts3Hash *pTerms){
-  int iTerm, nTerms, nResultBytes, iByte;
-  char *result;
-  TermData *pData;
-  fts3HashElem *e;
-
-  /* Iterate pTerms to generate an array of terms in pData for
-  ** sorting.
-  */
-  nTerms = fts3HashCount(pTerms);
-  assert( nTerms>0 );
-  pData = sqlite3_malloc(nTerms*sizeof(TermData));
-  if( pData==NULL ) return SQLITE_NOMEM;
-
-  nResultBytes = 0;
-  for(iTerm = 0, e = fts3HashFirst(pTerms); e; iTerm++, e = fts3HashNext(e)){
-    nResultBytes += fts3HashKeysize(e)+1;   /* Term plus trailing space */
-    assert( iTerm<nTerms );
-    pData[iTerm].pTerm = fts3HashKey(e);
-    pData[iTerm].nTerm = fts3HashKeysize(e);
-    pData[iTerm].pCollector = fts3HashData(e);  /* unused */
-  }
-  assert( iTerm==nTerms );
-
-  assert( nResultBytes>0 );   /* nTerms>0, nResultsBytes must be, too. */
-  result = sqlite3_malloc(nResultBytes);
-  if( result==NULL ){
-    sqlite3_free(pData);
-    return SQLITE_NOMEM;
-  }
-
-  if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp);
-
-  /* Read the terms in order to build the result. */
-  iByte = 0;
-  for(iTerm=0; iTerm<nTerms; ++iTerm){
-    memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm);
-    iByte += pData[iTerm].nTerm;
-    result[iByte++] = ' ';
-  }
-  assert( iByte==nResultBytes );
-  assert( result[nResultBytes-1]==' ' );
-  result[nResultBytes-1] = '\0';
-
-  /* Passes away ownership of result. */
-  sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free);
-  sqlite3_free(pData);
-  return SQLITE_OK;
-}
-
-/* Implements dump_terms() for use in inspecting the fts3 index from
-** tests.  TEXT result containing the ordered list of terms joined by
-** spaces.  dump_terms(t, level, idx) dumps the terms for the segment
-** specified by level, idx (in %_segdir), while dump_terms(t) dumps
-** all terms in the index.  In both cases t is the fts table's magic
-** table-named column.
+/*
+** Implementation of the offsets() function for FTS3
 */
-static void dumpTermsFunc(
-  sqlite3_context *pContext,
-  int argc, sqlite3_value **argv
+static void fts3OffsetsFunc(
+  sqlite3_context *pContext,      /* SQLite function call context */
+  int nVal,                       /* Size of argument array */
+  sqlite3_value **apVal           /* Array of arguments */
 ){
-  fulltext_cursor *pCursor;
-  if( argc!=3 && argc!=1 ){
-    generateError(pContext, "dump_terms", "incorrect arguments");
-  }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
-            sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
-    generateError(pContext, "dump_terms", "illegal first argument");
-  }else{
-    fulltext_vtab *v;
-    fts3Hash terms;
-    sqlite3_stmt *s = NULL;
-    int rc;
-
-    memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
-    v = cursor_vtab(pCursor);
+  Fts3Cursor *pCsr;               /* Cursor handle passed through apVal[0] */
 
-    /* If passed only the cursor column, get all segments.  Otherwise
-    ** get the segment described by the following two arguments.
-    */
-    if( argc==1 ){
-      rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
-    }else{
-      rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
-      if( rc==SQLITE_OK ){
-        rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1]));
-        if( rc==SQLITE_OK ){
-          rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2]));
-        }
-      }
-    }
+  UNUSED_PARAMETER(nVal);
 
-    if( rc!=SQLITE_OK ){
-      generateError(pContext, "dump_terms", NULL);
-      return;
-    }
-
-    /* Collect the terms for each segment. */
-    sqlite3Fts3HashInit(&terms, FTS3_HASH_STRING, 1);
-    while( (rc = sqlite3_step(s))==SQLITE_ROW ){
-      rc = collectSegmentTerms(v, s, &terms);
-      if( rc!=SQLITE_OK ) break;
-    }
-
-    if( rc!=SQLITE_DONE ){
-      sqlite3_reset(s);
-      generateError(pContext, "dump_terms", NULL);
-    }else{
-      const int nTerms = fts3HashCount(&terms);
-      if( nTerms>0 ){
-        rc = generateTermsResult(pContext, &terms);
-        if( rc==SQLITE_NOMEM ){
-          generateError(pContext, "dump_terms", "out of memory");
-        }else{
-          assert( rc==SQLITE_OK );
-        }
-      }else if( argc==3 ){
-        /* The specific segment asked for could not be found. */
-        generateError(pContext, "dump_terms", "segment not found");
-      }else{
-        /* No segments found. */
-        /* TODO(shess): It should be impossible to reach this.  This
-        ** case can only happen for an empty table, in which case
-        ** SQLite has no rows to call this function on.
-        */
-        sqlite3_result_null(pContext);
-      }
-    }
-    sqlite3Fts3HashClear(&terms);
+  assert( nVal==1 );
+  if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return;
+  assert( pCsr );
+  if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){
+    sqlite3Fts3Offsets(pContext, pCsr);
   }
 }
 
-/* Expand the DL_DEFAULT doclist in pData into a text result in
-** pContext.
+/* 
+** Implementation of the special optimize() function for FTS3. This 
+** function merges all segments in the database to a single segment.
+** Example usage is:
+**
+**   SELECT optimize(t) FROM t LIMIT 1;
+**
+** where 't' is the name of an FTS3 table.
 */
-static void createDoclistResult(sqlite3_context *pContext,
-                                const char *pData, int nData){
-  DataBuffer dump;
-  DLReader dlReader;
-  int rc;
+static void fts3OptimizeFunc(
+  sqlite3_context *pContext,      /* SQLite function call context */
+  int nVal,                       /* Size of argument array */
+  sqlite3_value **apVal           /* Array of arguments */
+){
+  int rc;                         /* Return code */
+  Fts3Table *p;                   /* Virtual table handle */
+  Fts3Cursor *pCursor;            /* Cursor handle passed through apVal[0] */
 
-  assert( pData!=NULL && nData>0 );
+  UNUSED_PARAMETER(nVal);
 
-  rc = dlrInit(&dlReader, DL_DEFAULT, pData, nData);
-  if( rc!=SQLITE_OK ) return rc;
-  dataBufferInit(&dump, 0);
-  for( ; rc==SQLITE_OK && !dlrAtEnd(&dlReader); rc = dlrStep(&dlReader) ){
-    char buf[256];
-    PLReader plReader;
+  assert( nVal==1 );
+  if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return;
+  p = (Fts3Table *)pCursor->base.pVtab;
+  assert( p );
 
-    rc = plrInit(&plReader, &dlReader);
-    if( rc!=SQLITE_OK ) break;
-    if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){
-      sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
-      dataBufferAppend(&dump, buf, strlen(buf));
-    }else{
-      int iColumn = plrColumn(&plReader);
-
-      sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[",
-                       dlrDocid(&dlReader), iColumn);
-      dataBufferAppend(&dump, buf, strlen(buf));
-
-      for( ; !plrAtEnd(&plReader); rc = plrStep(&plReader) ){
-        if( rc!=SQLITE_OK ) break;
-        if( plrColumn(&plReader)!=iColumn ){
-          iColumn = plrColumn(&plReader);
-          sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
-          assert( dump.nData>0 );
-          dump.nData--;                     /* Overwrite trailing space. */
-          assert( dump.pData[dump.nData]==' ');
-          dataBufferAppend(&dump, buf, strlen(buf));
-        }
-        if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){
-          sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ",
-                           plrPosition(&plReader),
-                           plrStartOffset(&plReader), plrEndOffset(&plReader));
-        }else if( DL_DEFAULT==DL_POSITIONS ){
-          sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader));
-        }else{
-          assert( NULL=="Unhandled DL_DEFAULT value");
-        }
-        dataBufferAppend(&dump, buf, strlen(buf));
-      }
-      plrDestroy(&plReader);
-      if( rc!= SQLITE_OK ) break;
+  rc = sqlite3Fts3Optimize(p);
 
-      assert( dump.nData>0 );
-      dump.nData--;                     /* Overwrite trailing space. */
-      assert( dump.pData[dump.nData]==' ');
-      dataBufferAppend(&dump, "]] ", 3);
-    }
-  }
-  dlrDestroy(&dlReader);
-  if( rc!=SQLITE_OK ){
-    dataBufferDestroy(&dump);
-    return rc;
+  switch( rc ){
+    case SQLITE_OK:
+      sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
+      break;
+    case SQLITE_DONE:
+      sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC);
+      break;
+    default:
+      sqlite3_result_error_code(pContext, rc);
+      break;
   }
-
-  assert( dump.nData>0 );
-  dump.nData--;                     /* Overwrite trailing space. */
-  assert( dump.pData[dump.nData]==' ');
-  dump.pData[dump.nData] = '\0';
-  assert( dump.nData>0 );
-
-  /* Passes ownership of dump's buffer to pContext. */
-  sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
-  dump.pData = NULL;
-  dump.nData = dump.nCapacity = 0;
-  return SQLITE_OK;
 }
 
-/* Implements dump_doclist() for use in inspecting the fts3 index from
-** tests.  TEXT result containing a string representation of the
-** doclist for the indicated term.  dump_doclist(t, term, level, idx)
-** dumps the doclist for term from the segment specified by level, idx
-** (in %_segdir), while dump_doclist(t, term) dumps the logical
-** doclist for the term across all segments.  The per-segment doclist
-** can contain deletions, while the full-index doclist will not
-** (deletions are omitted).
-**
-** Result formats differ with the setting of DL_DEFAULTS.  Examples:
-**
-** DL_DOCIDS: [1] [3] [7]
-** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]]
-** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]]
-**
-** In each case the number after the outer '[' is the docid.  In the
-** latter two cases, the number before the inner '[' is the column
-** associated with the values within.  For DL_POSITIONS the numbers
-** within are the positions, for DL_POSITIONS_OFFSETS they are the
-** position, the start offset, and the end offset.
+/*
+** Implementation of the matchinfo() function for FTS3
 */
-static void dumpDoclistFunc(
-  sqlite3_context *pContext,
-  int argc, sqlite3_value **argv
+static void fts3MatchinfoFunc(
+  sqlite3_context *pContext,      /* SQLite function call context */
+  int nVal,                       /* Size of argument array */
+  sqlite3_value **apVal           /* Array of arguments */
 ){
-  fulltext_cursor *pCursor;
-  if( argc!=2 && argc!=4 ){
-    generateError(pContext, "dump_doclist", "incorrect arguments");
-  }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
-            sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
-    generateError(pContext, "dump_doclist", "illegal first argument");
-  }else if( sqlite3_value_text(argv[1])==NULL ||
-            sqlite3_value_text(argv[1])[0]=='\0' ){
-    generateError(pContext, "dump_doclist", "empty second argument");
-  }else{
-    const char *pTerm = (const char *)sqlite3_value_text(argv[1]);
-    const int nTerm = strlen(pTerm);
-    fulltext_vtab *v;
-    int rc;
-    DataBuffer doclist;
-
-    memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
-    v = cursor_vtab(pCursor);
-
-    dataBufferInit(&doclist, 0);
-
-    /* termSelect() yields the same logical doclist that queries are
-    ** run against.
-    */
-    if( argc==2 ){
-      rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist);
-    }else{
-      sqlite3_stmt *s = NULL;
-
-      /* Get our specific segment's information. */
-      rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
-      if( rc==SQLITE_OK ){
-        rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2]));
-        if( rc==SQLITE_OK ){
-          rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3]));
-        }
-      }
-
-      if( rc==SQLITE_OK ){
-        rc = sqlite3_step(s);
-
-        if( rc==SQLITE_DONE ){
-          dataBufferDestroy(&doclist);
-          generateError(pContext, "dump_doclist", "segment not found");
-          return;
-        }
-
-        /* Found a segment, load it into doclist. */
-        if( rc==SQLITE_ROW ){
-          const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
-          const char *pData = sqlite3_column_blob(s, 2);
-          const int nData = sqlite3_column_bytes(s, 2);
-
-          /* loadSegment() is used by termSelect() to load each
-          ** segment's data.
-          */
-          rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0,
-                           &doclist);
-          if( rc==SQLITE_OK ){
-            rc = sqlite3_step(s);
-
-            /* Should not have more than one matching segment. */
-            if( rc!=SQLITE_DONE ){
-              sqlite3_reset(s);
-              dataBufferDestroy(&doclist);
-              generateError(pContext, "dump_doclist", "invalid segdir");
-              return;
-            }
-            rc = SQLITE_OK;
-          }
-        }
-      }
-
-      sqlite3_reset(s);
-    }
+  Fts3Cursor *pCsr;               /* Cursor handle passed through apVal[0] */
 
-    if( rc==SQLITE_OK ){
-      if( doclist.nData>0 ){
-        createDoclistResult(pContext, doclist.pData, doclist.nData);
-      }else{
-        /* TODO(shess): This can happen if the term is not present, or
-        ** if all instances of the term have been deleted and this is
-        ** an all-index dump.  It may be interesting to distinguish
-        ** these cases.
-        */
-        sqlite3_result_text(pContext, "", 0, SQLITE_STATIC);
-      }
-    }else if( rc==SQLITE_NOMEM ){
-      /* Handle out-of-memory cases specially because if they are
-      ** generated in fts3 code they may not be reflected in the db
-      ** handle.
-      */
-      /* TODO(shess): Handle this more comprehensively.
-      ** sqlite3ErrStr() has what I need, but is internal.
-      */
-      generateError(pContext, "dump_doclist", "out of memory");
-    }else{
-      generateError(pContext, "dump_doclist", NULL);
-    }
+  if( nVal!=1 ){
+    sqlite3_result_error(pContext,
+        "wrong number of arguments to function matchinfo()", -1);
+    return;
+  }
 
-    dataBufferDestroy(&doclist);
+  if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){
+    sqlite3Fts3Matchinfo(pContext, pCsr);
   }
 }
-#endif
 
 /*
 ** This routine implements the xFindFunction method for the FTS3
 ** virtual table.
 */
-static int fulltextFindFunction(
-  sqlite3_vtab *pVtab,
-  int nArg,
-  const char *zName,
-  void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
-  void **ppArg
+static int fts3FindFunctionMethod(
+  sqlite3_vtab *pVtab,            /* Virtual table handle */
+  int nArg,                       /* Number of SQL function arguments */
+  const char *zName,              /* Name of SQL function */
+  void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
+  void **ppArg                    /* Unused */
 ){
-  if( strcmp(zName,"snippet")==0 ){
-    *pxFunc = snippetFunc;
-    return 1;
-  }else if( strcmp(zName,"offsets")==0 ){
-    *pxFunc = snippetOffsetsFunc;
-    return 1;
-  }else if( strcmp(zName,"optimize")==0 ){
-    *pxFunc = optimizeFunc;
-    return 1;
-#ifdef SQLITE_TEST
-    /* NOTE(shess): These functions are present only for testing
-    ** purposes.  No particular effort is made to optimize their
-    ** execution or how they build their results.
-    */
-  }else if( strcmp(zName,"dump_terms")==0 ){
-    /* fprintf(stderr, "Found dump_terms\n"); */
-    *pxFunc = dumpTermsFunc;
-    return 1;
-  }else if( strcmp(zName,"dump_doclist")==0 ){
-    /* fprintf(stderr, "Found dump_doclist\n"); */
-    *pxFunc = dumpDoclistFunc;
-    return 1;
-#endif
+  struct Overloaded {
+    const char *zName;
+    void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
+  } aOverload[] = {
+    { "snippet", fts3SnippetFunc },
+    { "offsets", fts3OffsetsFunc },
+    { "optimize", fts3OptimizeFunc },
+    { "matchinfo", fts3MatchinfoFunc },
+  };
+  int i;                          /* Iterator variable */
+
+  UNUSED_PARAMETER(pVtab);
+  UNUSED_PARAMETER(nArg);
+  UNUSED_PARAMETER(ppArg);
+
+  for(i=0; i<SizeofArray(aOverload); i++){
+    if( strcmp(zName, aOverload[i].zName)==0 ){
+      *pxFunc = aOverload[i].xFunc;
+      return 1;
+    }
   }
+
+  /* No function of the specified name was found. Return 0. */
   return 0;
 }
 
 /*
-** Rename an fts3 table.
+** Implementation of FTS3 xRename method. Rename an fts3 table.
 */
-static int fulltextRename(
-  sqlite3_vtab *pVtab,
-  const char *zName
+static int fts3RenameMethod(
+  sqlite3_vtab *pVtab,            /* Virtual table handle */
+  const char *zName               /* New name of table */
 ){
-  fulltext_vtab *p = (fulltext_vtab *)pVtab;
-  int rc = SQLITE_NOMEM;
-  char *zSql = sqlite3_mprintf(
-    "ALTER TABLE %Q.'%q_content'  RENAME TO '%q_content';"
-    "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
-    "ALTER TABLE %Q.'%q_segdir'   RENAME TO '%q_segdir';"
-    , p->zDb, p->zName, zName 
-    , p->zDb, p->zName, zName 
-    , p->zDb, p->zName, zName
+  Fts3Table *p = (Fts3Table *)pVtab;
+  sqlite3 *db = p->db;            /* Database connection */
+  int rc;                         /* Return Code */
+
+  rc = sqlite3Fts3PendingTermsFlush(p);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  fts3DbExec(&rc, db,
+    "ALTER TABLE %Q.'%q_content'  RENAME TO '%q_content';",
+    p->zDb, p->zName, zName
   );
-  if( zSql ){
-    rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
-    sqlite3_free(zSql);
+  if( rc==SQLITE_ERROR ) rc = SQLITE_OK;
+  if( p->bHasDocsize ){
+    fts3DbExec(&rc, db,
+      "ALTER TABLE %Q.'%q_docsize'  RENAME TO '%q_docsize';",
+      p->zDb, p->zName, zName
+    );
+    fts3DbExec(&rc, db,
+      "ALTER TABLE %Q.'%q_stat'  RENAME TO '%q_stat';",
+      p->zDb, p->zName, zName
+    );
   }
+  fts3DbExec(&rc, db,
+    "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';",
+    p->zDb, p->zName, zName
+  );
+  fts3DbExec(&rc, db,
+    "ALTER TABLE %Q.'%q_segdir'   RENAME TO '%q_segdir';",
+    p->zDb, p->zName, zName
+  );
   return rc;
 }
 
 static const sqlite3_module fts3Module = {
   /* iVersion      */ 0,
-  /* xCreate       */ fulltextCreate,
-  /* xConnect      */ fulltextConnect,
-  /* xBestIndex    */ fulltextBestIndex,
-  /* xDisconnect   */ fulltextDisconnect,
-  /* xDestroy      */ fulltextDestroy,
-  /* xOpen         */ fulltextOpen,
+  /* xCreate       */ fts3CreateMethod,
+  /* xConnect      */ fts3ConnectMethod,
+  /* xBestIndex    */ fts3BestIndexMethod,
+  /* xDisconnect   */ fts3DisconnectMethod,
+  /* xDestroy      */ fts3DestroyMethod,
+  /* xOpen         */ fts3OpenMethod,
   /* xClose        */ fulltextClose,
-  /* xFilter       */ fulltextFilter,
-  /* xNext         */ fulltextNext,
-  /* xEof          */ fulltextEof,
-  /* xColumn       */ fulltextColumn,
-  /* xRowid        */ fulltextRowid,
-  /* xUpdate       */ fulltextUpdate,
-  /* xBegin        */ fulltextBegin,
-  /* xSync         */ fulltextSync,
-  /* xCommit       */ fulltextCommit,
-  /* xRollback     */ fulltextRollback,
-  /* xFindFunction */ fulltextFindFunction,
-  /* xRename */       fulltextRename,
+  /* xFilter       */ fts3FilterMethod,
+  /* xNext         */ fts3NextMethod,
+  /* xEof          */ fts3EofMethod,
+  /* xColumn       */ fts3ColumnMethod,
+  /* xRowid        */ fts3RowidMethod,
+  /* xUpdate       */ fts3UpdateMethod,
+  /* xBegin        */ fts3BeginMethod,
+  /* xSync         */ fts3SyncMethod,
+  /* xCommit       */ fts3CommitMethod,
+  /* xRollback     */ fts3RollbackMethod,
+  /* xFindFunction */ fts3FindFunctionMethod,
+  /* xRename */       fts3RenameMethod,
 };
 
+/*
+** This function is registered as the module destructor (called when an
+** FTS3 enabled database connection is closed). It frees the memory
+** allocated for the tokenizer hash table.
+*/
 static void hashDestroy(void *p){
-  fts3Hash *pHash = (fts3Hash *)p;
+  Fts3Hash *pHash = (Fts3Hash *)p;
   sqlite3Fts3HashClear(pHash);
   sqlite3_free(pHash);
 }
@@ -7318,7 +2669,7 @@ static void hashDestroy(void *p){
 ** used to retrieve the respective implementations.
 **
 ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
-** to by the argument to point a the "simple" tokenizer implementation.
+** to by the argument to point to the "simple" tokenizer implementation.
 ** Function ...PorterTokenizerModule() sets *pModule to point to the
 ** porter tokenizer/stemmer implementation.
 */
@@ -7326,8 +2677,6 @@ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
 void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
 void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
 
-int sqlite3Fts3InitHashTable(sqlite3 *, fts3Hash *, const char *);
-
 /*
 ** Initialise the fts3 extension. If this extension is built as part
 ** of the sqlite library, then this function is called directly by
@@ -7336,19 +2685,20 @@ int sqlite3Fts3InitHashTable(sqlite3 *, fts3Hash *, const char *);
 */
 int sqlite3Fts3Init(sqlite3 *db){
   int rc = SQLITE_OK;
-  fts3Hash *pHash = 0;
+  Fts3Hash *pHash = 0;
   const sqlite3_tokenizer_module *pSimple = 0;
   const sqlite3_tokenizer_module *pPorter = 0;
-  const sqlite3_tokenizer_module *pIcu = 0;
 
-  sqlite3Fts3SimpleTokenizerModule(&pSimple);
-  sqlite3Fts3PorterTokenizerModule(&pPorter);
 #ifdef SQLITE_ENABLE_ICU
+  const sqlite3_tokenizer_module *pIcu = 0;
   sqlite3Fts3IcuTokenizerModule(&pIcu);
 #endif
 
+  sqlite3Fts3SimpleTokenizerModule(&pSimple);
+  sqlite3Fts3PorterTokenizerModule(&pPorter);
+
   /* Allocate and initialise the hash-table used to store tokenizers. */
-  pHash = sqlite3_malloc(sizeof(fts3Hash));
+  pHash = sqlite3_malloc(sizeof(Fts3Hash));
   if( !pHash ){
     rc = SQLITE_NOMEM;
   }else{
@@ -7359,14 +2709,18 @@ int sqlite3Fts3Init(sqlite3 *db){
   if( rc==SQLITE_OK ){
     if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
      || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) 
+#ifdef SQLITE_ENABLE_ICU
      || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
+#endif
     ){
       rc = SQLITE_NOMEM;
     }
   }
 
 #ifdef SQLITE_TEST
-  sqlite3Fts3ExprInitTestInterface(db);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts3ExprInitTestInterface(db);
+  }
 #endif
 
   /* Create the virtual table wrapper around the hash-table and overload 
@@ -7374,22 +2728,21 @@ int sqlite3Fts3Init(sqlite3 *db){
   ** module with sqlite.
   */
   if( SQLITE_OK==rc 
-#if CHROMIUM_FTS3_CHANGES && !SQLITE_TEST
-      /* fts3_tokenizer() disabled for security reasons. */
-#else
    && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
-#endif
    && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
-   && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
-   && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
-#ifdef SQLITE_TEST
-   && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1))
-   && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1))
-#endif
+   && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
+   && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", -1))
+   && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
   ){
-    return sqlite3_create_module_v2(
+    rc = sqlite3_create_module_v2(
         db, "fts3", &fts3Module, (void *)pHash, hashDestroy
     );
+    if( rc==SQLITE_OK ){
+      rc = sqlite3_create_module_v2(
+          db, "fts4", &fts3Module, (void *)pHash, 0
+      );
+    }
+    return rc;
   }
 
   /* An error has occurred. Delete the hash table and return the error code. */
@@ -7412,4 +2765,4 @@ int sqlite3_extension_init(
 }
 #endif
 
-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+#endif
diff --git a/third_party/sqlite/src/ext/fts3/fts3Int.h b/third_party/sqlite/src/ext/fts3/fts3Int.h
new file mode 100644
index 0000000..8ed31aed
--- /dev/null
+++ b/third_party/sqlite/src/ext/fts3/fts3Int.h
@@ -0,0 +1,326 @@
+/*
+** 2009 Nov 12
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+*/
+
+#ifndef _FTSINT_H
+#define _FTSINT_H
+
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
+# define NDEBUG 1
+#endif
+
+#include "sqlite3.h"
+#include "fts3_tokenizer.h"
+#include "fts3_hash.h"
+
+/*
+** This constant controls how often segments are merged. Once there are
+** FTS3_MERGE_COUNT segments of level N, they are merged into a single
+** segment of level N+1.
+*/
+#define FTS3_MERGE_COUNT 16
+
+/*
+** This is the maximum amount of data (in bytes) to store in the 
+** Fts3Table.pendingTerms hash table. Normally, the hash table is
+** populated as documents are inserted/updated/deleted in a transaction
+** and used to create a new segment when the transaction is committed.
+** However if this limit is reached midway through a transaction, a new 
+** segment is created and the hash table cleared immediately.
+*/
+#define FTS3_MAX_PENDING_DATA (1*1024*1024)
+
+/*
+** Macro to return the number of elements in an array. SQLite has a
+** similar macro called ArraySize(). Use a different name to avoid
+** a collision when building an amalgamation with built-in FTS3.
+*/
+#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
+
+/*
+** Maximum length of a varint encoded integer. The varint format is different
+** from that used by SQLite, so the maximum length is 10, not 9.
+*/
+#define FTS3_VARINT_MAX 10
+
+/*
+** The testcase() macro is only used by the amalgamation.  If undefined,
+** make it a no-op.
+*/
+#ifndef testcase
+# define testcase(X)
+#endif
+
+/*
+** Terminator values for position-lists and column-lists.
+*/
+#define POS_COLUMN  (1)     /* Column-list terminator */
+#define POS_END     (0)     /* Position-list terminator */ 
+
+/*
+** This section provides definitions to allow the
+** FTS3 extension to be compiled outside of the 
+** amalgamation.
+*/
+#ifndef SQLITE_AMALGAMATION
+/*
+** Macros indicating that conditional expressions are always true or
+** false.
+*/
+# define ALWAYS(x) (x)
+# define NEVER(X)  (x)
+/*
+** Internal types used by SQLite.
+*/
+typedef unsigned char u8;         /* 1-byte (or larger) unsigned integer */
+typedef short int i16;            /* 2-byte (or larger) signed integer */
+typedef unsigned int u32;         /* 4-byte unsigned integer */
+typedef sqlite3_uint64 u64;       /* 8-byte unsigned integer */
+/*
+** Macro used to suppress compiler warnings for unused parameters.
+*/
+#define UNUSED_PARAMETER(x) (void)(x)
+#endif
+
+typedef struct Fts3Table Fts3Table;
+typedef struct Fts3Cursor Fts3Cursor;
+typedef struct Fts3Expr Fts3Expr;
+typedef struct Fts3Phrase Fts3Phrase;
+typedef struct Fts3SegReader Fts3SegReader;
+typedef struct Fts3SegFilter Fts3SegFilter;
+
+/*
+** A connection to a fulltext index is an instance of the following
+** structure. The xCreate and xConnect methods create an instance
+** of this structure and xDestroy and xDisconnect free that instance.
+** All other methods receive a pointer to the structure as one of their
+** arguments.
+*/
+struct Fts3Table {
+  sqlite3_vtab base;              /* Base class used by SQLite core */
+  sqlite3 *db;                    /* The database connection */
+  const char *zDb;                /* logical database name */
+  const char *zName;              /* virtual table name */
+  int nColumn;                    /* number of named columns in virtual table */
+  char **azColumn;                /* column names.  malloced */
+  sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
+
+  /* Precompiled statements used by the implementation. Each of these 
+  ** statements is run and reset within a single virtual table API call. 
+  */
+  sqlite3_stmt *aStmt[25];
+
+  /* Pointer to string containing the SQL:
+  **
+  ** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ? 
+  **    ORDER BY blockid"
+  */
+  char *zSelectLeaves;
+  int nLeavesStmt;                /* Valid statements in aLeavesStmt */
+  int nLeavesTotal;               /* Total number of prepared leaves stmts */
+  int nLeavesAlloc;               /* Allocated size of aLeavesStmt */
+  sqlite3_stmt **aLeavesStmt;     /* Array of prepared zSelectLeaves stmts */
+
+  int nNodeSize;                  /* Soft limit for node size */
+  u8 bHasContent;                 /* True if %_content table exists */
+  u8 bHasDocsize;                 /* True if %_docsize table exists */
+
+  /* The following hash table is used to buffer pending index updates during
+  ** transactions. Variable nPendingData estimates the memory size of the 
+  ** pending data, including hash table overhead, but not malloc overhead. 
+  ** When nPendingData exceeds nMaxPendingData, the buffer is flushed 
+  ** automatically. Variable iPrevDocid is the docid of the most recently
+  ** inserted record.
+  */
+  int nMaxPendingData;
+  int nPendingData;
+  sqlite_int64 iPrevDocid;
+  Fts3Hash pendingTerms;
+};
+
+/*
+** When the core wants to read from the virtual table, it creates a
+** virtual table cursor (an instance of the following structure) using
+** the xOpen method. Cursors are destroyed using the xClose method.
+*/
+struct Fts3Cursor {
+  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
+  i16 eSearch;                    /* Search strategy (see below) */
+  u8 isEof;                       /* True if at End Of Results */
+  u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
+  sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
+  Fts3Expr *pExpr;                /* Parsed MATCH query string */
+  sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
+  char *pNextId;                  /* Pointer into the body of aDoclist */
+  char *aDoclist;                 /* List of docids for full-text queries */
+  int nDoclist;                   /* Size of buffer at aDoclist */
+  int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
+  u32 *aMatchinfo;                /* Information about most recent match */
+};
+
+/*
+** The Fts3Cursor.eSearch member is always set to one of the following.
+** Actualy, Fts3Cursor.eSearch can be greater than or equal to
+** FTS3_FULLTEXT_SEARCH.  If so, then Fts3Cursor.eSearch - 2 is the index
+** of the column to be searched.  For example, in
+**
+**     CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d);
+**     SELECT docid FROM ex1 WHERE b MATCH 'one two three';
+** 
+** Because the LHS of the MATCH operator is 2nd column "b",
+** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1.  (+0 for a,
+** +1 for b, +2 for c, +3 for d.)  If the LHS of MATCH were "ex1" 
+** indicating that all columns should be searched,
+** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4.
+*/
+#define FTS3_FULLSCAN_SEARCH 0    /* Linear scan of %_content table */
+#define FTS3_DOCID_SEARCH    1    /* Lookup by rowid on %_content table */
+#define FTS3_FULLTEXT_SEARCH 2    /* Full-text index search */
+
+/*
+** A "phrase" is a sequence of one or more tokens that must match in
+** sequence.  A single token is the base case and the most common case.
+** For a sequence of tokens contained in "...", nToken will be the number
+** of tokens in the string.
+*/
+struct Fts3Phrase {
+  int nToken;                /* Number of tokens in the phrase */
+  int iColumn;               /* Index of column this phrase must match */
+  int isNot;                 /* Phrase prefixed by unary not (-) operator */
+  struct PhraseToken {
+    char *z;                 /* Text of the token */
+    int n;                   /* Number of bytes in buffer pointed to by z */
+    int isPrefix;            /* True if token ends in with a "*" character */
+  } aToken[1];               /* One entry for each token in the phrase */
+};
+
+/*
+** A tree of these objects forms the RHS of a MATCH operator.
+**
+** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded
+** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, 
+** containing the results of the NEAR or phrase query in FTS3 doclist
+** format. As usual, the initial "Length" field found in doclists stored
+** on disk is omitted from this buffer.
+**
+** Variable pCurrent always points to the start of a docid field within
+** aDoclist. Since the doclist is usually scanned in docid order, this can
+** be used to accelerate seeking to the required docid within the doclist.
+*/
+struct Fts3Expr {
+  int eType;                 /* One of the FTSQUERY_XXX values defined below */
+  int nNear;                 /* Valid if eType==FTSQUERY_NEAR */
+  Fts3Expr *pParent;         /* pParent->pLeft==this or pParent->pRight==this */
+  Fts3Expr *pLeft;           /* Left operand */
+  Fts3Expr *pRight;          /* Right operand */
+  Fts3Phrase *pPhrase;       /* Valid if eType==FTSQUERY_PHRASE */
+
+  int isLoaded;              /* True if aDoclist/nDoclist are initialized. */
+  char *aDoclist;            /* Buffer containing doclist */
+  int nDoclist;              /* Size of aDoclist in bytes */
+
+  sqlite3_int64 iCurrent;
+  char *pCurrent;
+};
+
+/*
+** Candidate values for Fts3Query.eType. Note that the order of the first
+** four values is in order of precedence when parsing expressions. For 
+** example, the following:
+**
+**   "a OR b AND c NOT d NEAR e"
+**
+** is equivalent to:
+**
+**   "a OR (b AND (c NOT (d NEAR e)))"
+*/
+#define FTSQUERY_NEAR   1
+#define FTSQUERY_NOT    2
+#define FTSQUERY_AND    3
+#define FTSQUERY_OR     4
+#define FTSQUERY_PHRASE 5
+
+
+/* fts3_init.c */
+int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *);
+int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*, 
+                        sqlite3_vtab **, char **);
+
+/* fts3_write.c */
+int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
+int sqlite3Fts3PendingTermsFlush(Fts3Table *);
+void sqlite3Fts3PendingTermsClear(Fts3Table *);
+int sqlite3Fts3Optimize(Fts3Table *);
+int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
+  sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
+int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
+void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *);
+int sqlite3Fts3SegReaderIterate(
+  Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
+  int (*)(Fts3Table *, void *, char *, int, char *, int),  void *
+);
+int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*);
+int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
+int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*);
+int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*);
+int sqlite3Fts3ReadLock(Fts3Table *);
+
+/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
+#define FTS3_SEGMENT_REQUIRE_POS   0x00000001
+#define FTS3_SEGMENT_IGNORE_EMPTY  0x00000002
+#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
+#define FTS3_SEGMENT_PREFIX        0x00000008
+
+/* Type passed as 4th argument to SegmentReaderIterate() */
+struct Fts3SegFilter {
+  const char *zTerm;
+  int nTerm;
+  int iCol;
+  int flags;
+};
+
+/* fts3.c */
+int sqlite3Fts3PutVarint(char *, sqlite3_int64);
+int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
+int sqlite3Fts3GetVarint32(const char *, int *);
+int sqlite3Fts3VarintLen(sqlite3_uint64);
+void sqlite3Fts3Dequote(char *);
+
+char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int);
+int sqlite3Fts3ExprLoadDoclist(Fts3Table *, Fts3Expr *);
+int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
+
+/* fts3_tokenizer.c */
+const char *sqlite3Fts3NextToken(const char *, int *);
+int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
+int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, 
+  const char *, sqlite3_tokenizer **, const char **, char **
+);
+
+/* fts3_snippet.c */
+void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
+void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
+  const char *, const char *, int, int
+);
+void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *);
+
+/* fts3_expr.c */
+int sqlite3Fts3ExprParse(sqlite3_tokenizer *, 
+  char **, int, int, const char *, int, Fts3Expr **
+);
+void sqlite3Fts3ExprFree(Fts3Expr *);
+#ifdef SQLITE_TEST
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
+#endif
+
+#endif /* _FTSINT_H */
diff --git a/third_party/sqlite/src/ext/fts3/fts3_expr.c b/third_party/sqlite/src/ext/fts3/fts3_expr.c
index bfca3e1..008ba81 100644
--- a/third_party/sqlite/src/ext/fts3/fts3_expr.c
+++ b/third_party/sqlite/src/ext/fts3/fts3_expr.c
@@ -13,8 +13,7 @@
 ** This module contains code that implements a parser for fts3 query strings
 ** (the right-hand argument to the MATCH operator). Because the supported 
 ** syntax is relatively simple, the whole tokenizer/parser system is
-** hand-coded. The public interface to this module is declared in source
-** code file "fts3_expr.h".
+** hand-coded. 
 */
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
 
@@ -40,7 +39,29 @@
 ** to zero causes the module to use the old syntax. If it is set to 
 ** non-zero the new syntax is activated. This is so both syntaxes can
 ** be tested using a single build of testfixture.
+**
+** The following describes the syntax supported by the fts3 MATCH
+** operator in a similar format to that used by the lemon parser
+** generator. This module does not use actually lemon, it uses a
+** custom parser.
+**
+**   query ::= andexpr (OR andexpr)*.
+**
+**   andexpr ::= notexpr (AND? notexpr)*.
+**
+**   notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
+**   notexpr ::= LP query RP.
+**
+**   nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
+**
+**   distance_opt ::= .
+**   distance_opt ::= / INTEGER.
+**
+**   phrase ::= TOKEN.
+**   phrase ::= COLUMN:TOKEN.
+**   phrase ::= "TOKEN TOKEN TOKEN...".
 */
+
 #ifdef SQLITE_TEST
 int sqlite3_fts3_enable_parentheses = 0;
 #else
@@ -56,8 +77,7 @@ int sqlite3_fts3_enable_parentheses = 0;
 */
 #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
 
-#include "fts3_expr.h"
-#include "sqlite3.h"
+#include "fts3Int.h"
 #include <string.h>
 #include <assert.h>
 
@@ -160,7 +180,7 @@ static int getNextToken(
 ** Enlarge a memory allocation.  If an out-of-memory allocation occurs,
 ** then free the old allocation.
 */
-void *fts3ReallocOrFree(void *pOrig, int nNew){
+static void *fts3ReallocOrFree(void *pOrig, int nNew){
   void *pRet = sqlite3_realloc(pOrig, nNew);
   if( !pRet ){
     sqlite3_free(pOrig);
@@ -231,7 +251,7 @@ static int getNextString(
 
   if( rc==SQLITE_DONE ){
     int jj;
-    char *zNew;
+    char *zNew = NULL;
     int nNew = 0;
     int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
     nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken);
@@ -290,7 +310,7 @@ static int getNextNode(
   int *pnConsumed                         /* OUT: Number of bytes consumed */
 ){
   static const struct Fts3Keyword {
-    char z[4];                            /* Keyword text */
+    char *z;                              /* Keyword text */
     unsigned char n;                      /* Length of the keyword */
     unsigned char parenOnly;              /* Only valid in paren mode */
     unsigned char eType;                  /* Keyword code */
@@ -353,11 +373,14 @@ static int getNextNode(
        || cNext=='"' || cNext=='(' || cNext==')' || cNext==0
       ){
         pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
+        if( !pRet ){
+          return SQLITE_NOMEM;
+        }
         memset(pRet, 0, sizeof(Fts3Expr));
         pRet->eType = pKey->eType;
         pRet->nNear = nNear;
         *ppExpr = pRet;
-        *pnConsumed = (zInput - z) + nKey;
+        *pnConsumed = (int)((zInput - z) + nKey);
         return SQLITE_OK;
       }
 
@@ -377,14 +400,14 @@ static int getNextNode(
       if( rc==SQLITE_OK && !*ppExpr ){
         rc = SQLITE_DONE;
       }
-      *pnConsumed = (zInput - z) + 1 + nConsumed;
+      *pnConsumed = (int)((zInput - z) + 1 + nConsumed);
       return rc;
     }
   
     /* Check for a close bracket. */
     if( *zInput==')' ){
       pParse->nNest--;
-      *pnConsumed = (zInput - z) + 1;
+      *pnConsumed = (int)((zInput - z) + 1);
       return SQLITE_DONE;
     }
   }
@@ -396,7 +419,7 @@ static int getNextNode(
   */
   if( *zInput=='"' ){
     for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);
-    *pnConsumed = (zInput - z) + ii + 1;
+    *pnConsumed = (int)((zInput - z) + ii + 1);
     if( ii==nInput ){
       return SQLITE_ERROR;
     }
@@ -419,12 +442,12 @@ static int getNextNode(
   iColLen = 0;
   for(ii=0; ii<pParse->nCol; ii++){
     const char *zStr = pParse->azCol[ii];
-    int nStr = strlen(zStr);
+    int nStr = (int)strlen(zStr);
     if( nInput>nStr && zInput[nStr]==':' 
      && sqlite3_strnicmp(zStr, zInput, nStr)==0 
     ){
       iCol = ii;
-      iColLen = ((zInput - z) + nStr + 1);
+      iColLen = (int)((zInput - z) + nStr + 1);
       break;
     }
   }
@@ -690,7 +713,7 @@ int sqlite3Fts3ExprParse(
     return SQLITE_OK;
   }
   if( n<0 ){
-    n = strlen(z);
+    n = (int)strlen(z);
   }
   rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed);
 
@@ -711,6 +734,7 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){
   if( p ){
     sqlite3Fts3ExprFree(p->pLeft);
     sqlite3Fts3ExprFree(p->pRight);
+    sqlite3_free(p->aDoclist);
     sqlite3_free(p);
   }
 }
@@ -745,7 +769,7 @@ static int queryTestTokenizer(
   sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
   if( SQLITE_ROW==sqlite3_step(pStmt) ){
     if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
-      memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
     }
   }
 
@@ -889,8 +913,8 @@ exprtest_out:
 ** Register the query expression parser test function fts3_exprtest() 
 ** with database connection db. 
 */
-void sqlite3Fts3ExprInitTestInterface(sqlite3* db){
-  sqlite3_create_function(
+int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
+  return sqlite3_create_function(
       db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
   );
 }
diff --git a/third_party/sqlite/src/ext/fts3/fts3_expr.h b/third_party/sqlite/src/ext/fts3/fts3_expr.h
deleted file mode 100644
index a48dee6..0000000
--- a/third_party/sqlite/src/ext/fts3/fts3_expr.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
-** 2008 Nov 28
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-*/
-
-#include "fts3_tokenizer.h"
-#include "sqlite3.h"
-
-/*
-** The following describes the syntax supported by the fts3 MATCH
-** operator in a similar format to that used by the lemon parser
-** generator. This module does not use actually lemon, it uses a
-** custom parser.
-**
-**   query ::= andexpr (OR andexpr)*.
-**
-**   andexpr ::= notexpr (AND? notexpr)*.
-**
-**   notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
-**   notexpr ::= LP query RP.
-**
-**   nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
-**
-**   distance_opt ::= .
-**   distance_opt ::= / INTEGER.
-**
-**   phrase ::= TOKEN.
-**   phrase ::= COLUMN:TOKEN.
-**   phrase ::= "TOKEN TOKEN TOKEN...".
-*/
-
-typedef struct Fts3Expr Fts3Expr;
-typedef struct Fts3Phrase Fts3Phrase;
-
-/*
-** A "phrase" is a sequence of one or more tokens that must match in
-** sequence.  A single token is the base case and the most common case.
-** For a sequence of tokens contained in "...", nToken will be the number
-** of tokens in the string.
-*/
-struct Fts3Phrase {
-  int nToken;          /* Number of tokens in the phrase */
-  int iColumn;         /* Index of column this phrase must match */
-  int isNot;           /* Phrase prefixed by unary not (-) operator */
-  struct PhraseToken {
-    char *z;              /* Text of the token */
-    int n;                /* Number of bytes in buffer pointed to by z */
-    int isPrefix;         /* True if token ends in with a "*" character */
-  } aToken[1];         /* One entry for each token in the phrase */
-};
-
-/*
-** A tree of these objects forms the RHS of a MATCH operator.
-*/
-struct Fts3Expr {
-  int eType;                 /* One of the FTSQUERY_XXX values defined below */
-  int nNear;                 /* Valid if eType==FTSQUERY_NEAR */
-  Fts3Expr *pParent;         /* pParent->pLeft==this or pParent->pRight==this */
-  Fts3Expr *pLeft;           /* Left operand */
-  Fts3Expr *pRight;          /* Right operand */
-  Fts3Phrase *pPhrase;       /* Valid if eType==FTSQUERY_PHRASE */
-};
-
-int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int, 
-                         const char *, int, Fts3Expr **);
-void sqlite3Fts3ExprFree(Fts3Expr *);
-
-/*
-** Candidate values for Fts3Query.eType. Note that the order of the first
-** four values is in order of precedence when parsing expressions. For 
-** example, the following:
-**
-**   "a OR b AND c NOT d NEAR e"
-**
-** is equivalent to:
-**
-**   "a OR (b AND (c NOT (d NEAR e)))"
-*/
-#define FTSQUERY_NEAR   1
-#define FTSQUERY_NOT    2
-#define FTSQUERY_AND    3
-#define FTSQUERY_OR     4
-#define FTSQUERY_PHRASE 5
-
-#ifdef SQLITE_TEST
-void sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
-#endif
diff --git a/third_party/sqlite/src/ext/fts3/fts3_hash.c b/third_party/sqlite/src/ext/fts3/fts3_hash.c
index ee30117..98be529 100644
--- a/third_party/sqlite/src/ext/fts3/fts3_hash.c
+++ b/third_party/sqlite/src/ext/fts3/fts3_hash.c
@@ -56,7 +56,7 @@ static void fts3HashFree(void *p){
 ** true if the hash table should make its own private copy of keys and
 ** false if it should just use the supplied pointer.
 */
-void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){
+void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){
   assert( pNew!=0 );
   assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
   pNew->keyClass = keyClass;
@@ -71,8 +71,8 @@ void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){
 ** Call this routine to delete a hash table or to reset a hash table
 ** to the empty state.
 */
-void sqlite3Fts3HashClear(fts3Hash *pH){
-  fts3HashElem *elem;         /* For looping over all elements of the table */
+void sqlite3Fts3HashClear(Fts3Hash *pH){
+  Fts3HashElem *elem;         /* For looping over all elements of the table */
 
   assert( pH!=0 );
   elem = pH->first;
@@ -81,7 +81,7 @@ void sqlite3Fts3HashClear(fts3Hash *pH){
   pH->ht = 0;
   pH->htsize = 0;
   while( elem ){
-    fts3HashElem *next_elem = elem->next;
+    Fts3HashElem *next_elem = elem->next;
     if( pH->copyKey && elem->pKey ){
       fts3HashFree(elem->pKey);
     }
@@ -164,11 +164,11 @@ static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){
 /* Link an element into the hash table
 */
 static void fts3HashInsertElement(
-  fts3Hash *pH,            /* The complete hash table */
+  Fts3Hash *pH,            /* The complete hash table */
   struct _fts3ht *pEntry,  /* The entry into which pNew is inserted */
-  fts3HashElem *pNew       /* The element to be inserted */
+  Fts3HashElem *pNew       /* The element to be inserted */
 ){
-  fts3HashElem *pHead;     /* First element already in pEntry */
+  Fts3HashElem *pHead;     /* First element already in pEntry */
   pHead = pEntry->chain;
   if( pHead ){
     pNew->next = pHead;
@@ -190,15 +190,17 @@ static void fts3HashInsertElement(
 /* Resize the hash table so that it cantains "new_size" buckets.
 ** "new_size" must be a power of 2.  The hash table might fail 
 ** to resize if sqliteMalloc() fails.
+**
+** Return non-zero if a memory allocation error occurs.
 */
-static void fts3Rehash(fts3Hash *pH, int new_size){
+static int fts3Rehash(Fts3Hash *pH, int new_size){
   struct _fts3ht *new_ht;          /* The new hash table */
-  fts3HashElem *elem, *next_elem;  /* For looping over existing elements */
+  Fts3HashElem *elem, *next_elem;  /* For looping over existing elements */
   int (*xHash)(const void*,int);   /* The hash function */
 
   assert( (new_size & (new_size-1))==0 );
   new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) );
-  if( new_ht==0 ) return;
+  if( new_ht==0 ) return 1;
   fts3HashFree(pH->ht);
   pH->ht = new_ht;
   pH->htsize = new_size;
@@ -208,19 +210,20 @@ static void fts3Rehash(fts3Hash *pH, int new_size){
     next_elem = elem->next;
     fts3HashInsertElement(pH, &new_ht[h], elem);
   }
+  return 0;
 }
 
 /* This function (for internal use only) locates an element in an
 ** hash table that matches the given key.  The hash for this key has
 ** already been computed and is passed as the 4th parameter.
 */
-static fts3HashElem *fts3FindElementByHash(
-  const fts3Hash *pH, /* The pH to be searched */
+static Fts3HashElem *fts3FindElementByHash(
+  const Fts3Hash *pH, /* The pH to be searched */
   const void *pKey,   /* The key we are searching for */
   int nKey,
   int h               /* The hash for this key. */
 ){
-  fts3HashElem *elem;            /* Used to loop thru the element list */
+  Fts3HashElem *elem;            /* Used to loop thru the element list */
   int count;                     /* Number of elements left to test */
   int (*xCompare)(const void*,int,const void*,int);  /* comparison function */
 
@@ -243,8 +246,8 @@ static fts3HashElem *fts3FindElementByHash(
 ** element and a hash on the element's key.
 */
 static void fts3RemoveElementByHash(
-  fts3Hash *pH,         /* The pH containing "elem" */
-  fts3HashElem* elem,   /* The element to be removed from the pH */
+  Fts3Hash *pH,         /* The pH containing "elem" */
+  Fts3HashElem* elem,   /* The element to be removed from the pH */
   int h                 /* Hash value for the element */
 ){
   struct _fts3ht *pEntry;
@@ -276,13 +279,12 @@ static void fts3RemoveElementByHash(
   }
 }
 
-/* Attempt to locate an element of the hash table pH with a key
-** that matches pKey,nKey.  Return the data for this element if it is
-** found, or NULL if there is no match.
-*/
-void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){
-  int h;                 /* A hash on key */
-  fts3HashElem *elem;    /* The element that matches key */
+Fts3HashElem *sqlite3Fts3HashFindElem(
+  const Fts3Hash *pH, 
+  const void *pKey, 
+  int nKey
+){
+  int h;                          /* A hash on key */
   int (*xHash)(const void*,int);  /* The hash function */
 
   if( pH==0 || pH->ht==0 ) return 0;
@@ -290,8 +292,19 @@ void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){
   assert( xHash!=0 );
   h = (*xHash)(pKey,nKey);
   assert( (pH->htsize & (pH->htsize-1))==0 );
-  elem = fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
-  return elem ? elem->data : 0;
+  return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
+}
+
+/* 
+** Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey.  Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){
+  Fts3HashElem *pElem;            /* The element that matches key (if any) */
+
+  pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey);
+  return pElem ? pElem->data : 0;
 }
 
 /* Insert an element into the hash table pH.  The key is pKey,nKey
@@ -310,15 +323,15 @@ void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){
 ** element corresponding to "key" is removed from the hash table.
 */
 void *sqlite3Fts3HashInsert(
-  fts3Hash *pH,        /* The hash table to insert into */
+  Fts3Hash *pH,        /* The hash table to insert into */
   const void *pKey,    /* The key */
   int nKey,            /* Number of bytes in the key */
   void *data           /* The data */
 ){
   int hraw;                 /* Raw hash value of the key */
   int h;                    /* the hash of the key modulo hash table size */
-  fts3HashElem *elem;       /* Used to loop thru the element list */
-  fts3HashElem *new_elem;   /* New element added to the pH */
+  Fts3HashElem *elem;       /* Used to loop thru the element list */
+  Fts3HashElem *new_elem;   /* New element added to the pH */
   int (*xHash)(const void*,int);  /* The hash function */
 
   assert( pH!=0 );
@@ -338,14 +351,14 @@ void *sqlite3Fts3HashInsert(
     return old_data;
   }
   if( data==0 ) return 0;
-  if( pH->htsize==0 ){
-    fts3Rehash(pH,8);
-    if( pH->htsize==0 ){
-      pH->count = 0;
-      return data;
-    }
+  if( (pH->htsize==0 && fts3Rehash(pH,8))
+   || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2))
+  ){
+    pH->count = 0;
+    return data;
   }
-  new_elem = (fts3HashElem*)fts3HashMalloc( sizeof(fts3HashElem) );
+  assert( pH->htsize>0 );
+  new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) );
   if( new_elem==0 ) return data;
   if( pH->copyKey && pKey!=0 ){
     new_elem->pKey = fts3HashMalloc( nKey );
@@ -359,9 +372,6 @@ void *sqlite3Fts3HashInsert(
   }
   new_elem->nKey = nKey;
   pH->count++;
-  if( pH->count > pH->htsize ){
-    fts3Rehash(pH,pH->htsize*2);
-  }
   assert( pH->htsize>0 );
   assert( (pH->htsize & (pH->htsize-1))==0 );
   h = hraw & (pH->htsize-1);
diff --git a/third_party/sqlite/src/ext/fts3/fts3_hash.h b/third_party/sqlite/src/ext/fts3/fts3_hash.h
index e01954e..399f515 100644
--- a/third_party/sqlite/src/ext/fts3/fts3_hash.h
+++ b/third_party/sqlite/src/ext/fts3/fts3_hash.h
@@ -18,8 +18,8 @@
 #define _FTS3_HASH_H_
 
 /* Forward declarations of structures. */
-typedef struct fts3Hash fts3Hash;
-typedef struct fts3HashElem fts3HashElem;
+typedef struct Fts3Hash Fts3Hash;
+typedef struct Fts3HashElem Fts3HashElem;
 
 /* A complete hash table is an instance of the following structure.
 ** The internals of this structure are intended to be opaque -- client
@@ -29,15 +29,15 @@ typedef struct fts3HashElem fts3HashElem;
 ** accessing this structure are really macros, so we can't really make
 ** this structure opaque.
 */
-struct fts3Hash {
+struct Fts3Hash {
   char keyClass;          /* HASH_INT, _POINTER, _STRING, _BINARY */
   char copyKey;           /* True if copy of key made on insert */
   int count;              /* Number of entries in this table */
-  fts3HashElem *first;    /* The first element of the array */
+  Fts3HashElem *first;    /* The first element of the array */
   int htsize;             /* Number of buckets in the hash table */
   struct _fts3ht {        /* the hash table */
     int count;               /* Number of entries with this hash */
-    fts3HashElem *chain;     /* Pointer to first entry with this hash */
+    Fts3HashElem *chain;     /* Pointer to first entry with this hash */
   } *ht;
 };
 
@@ -47,8 +47,8 @@ struct fts3Hash {
 ** Again, this structure is intended to be opaque, but it can't really
 ** be opaque because it is used by macros.
 */
-struct fts3HashElem {
-  fts3HashElem *next, *prev; /* Next and previous elements in the table */
+struct Fts3HashElem {
+  Fts3HashElem *next, *prev; /* Next and previous elements in the table */
   void *data;                /* Data associated with this element */
   void *pKey; int nKey;      /* Key associated with this element */
 };
@@ -71,25 +71,27 @@ struct fts3HashElem {
 /*
 ** Access routines.  To delete, insert a NULL pointer.
 */
-void sqlite3Fts3HashInit(fts3Hash*, int keytype, int copyKey);
-void *sqlite3Fts3HashInsert(fts3Hash*, const void *pKey, int nKey, void *pData);
-void *sqlite3Fts3HashFind(const fts3Hash*, const void *pKey, int nKey);
-void sqlite3Fts3HashClear(fts3Hash*);
+void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey);
+void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData);
+void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey);
+void sqlite3Fts3HashClear(Fts3Hash*);
+Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int);
 
 /*
 ** Shorthand for the functions above
 */
-#define fts3HashInit   sqlite3Fts3HashInit
-#define fts3HashInsert sqlite3Fts3HashInsert
-#define fts3HashFind   sqlite3Fts3HashFind
-#define fts3HashClear  sqlite3Fts3HashClear
+#define fts3HashInit     sqlite3Fts3HashInit
+#define fts3HashInsert   sqlite3Fts3HashInsert
+#define fts3HashFind     sqlite3Fts3HashFind
+#define fts3HashClear    sqlite3Fts3HashClear
+#define fts3HashFindElem sqlite3Fts3HashFindElem
 
 /*
 ** Macros for looping over all elements of a hash table.  The idiom is
 ** like this:
 **
-**   fts3Hash h;
-**   fts3HashElem *p;
+**   Fts3Hash h;
+**   Fts3HashElem *p;
 **   ...
 **   for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
 **     SomeStructure *pData = fts3HashData(p);
diff --git a/third_party/sqlite/src/ext/fts3/fts3_porter.c b/third_party/sqlite/src/ext/fts3/fts3_porter.c
index 001060c..5963abc 100644
--- a/third_party/sqlite/src/ext/fts3/fts3_porter.c
+++ b/third_party/sqlite/src/ext/fts3/fts3_porter.c
@@ -24,6 +24,7 @@
 */
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
 
+#include "fts3Int.h"
 
 #include <assert.h>
 #include <stdlib.h>
@@ -53,10 +54,6 @@ typedef struct porter_tokenizer_cursor {
 } porter_tokenizer_cursor;
 
 
-/* Forward declaration */
-static const sqlite3_tokenizer_module porterTokenizerModule;
-
-
 /*
 ** Create a new tokenizer instance.
 */
@@ -65,6 +62,10 @@ static int porterCreate(
   sqlite3_tokenizer **ppTokenizer
 ){
   porter_tokenizer *t;
+
+  UNUSED_PARAMETER(argc);
+  UNUSED_PARAMETER(argv);
+
   t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
   if( t==NULL ) return SQLITE_NOMEM;
   memset(t, 0, sizeof(*t));
@@ -93,6 +94,8 @@ static int porterOpen(
 ){
   porter_tokenizer_cursor *c;
 
+  UNUSED_PARAMETER(pTokenizer);
+
   c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
   if( c==NULL ) return SQLITE_NOMEM;
 
@@ -233,7 +236,7 @@ static int hasVowel(const char *z){
 ** the first two characters of z[].
 */
 static int doubleConsonant(const char *z){
-  return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
+  return isConsonant(z) && z[0]==z[1];
 }
 
 /*
@@ -246,10 +249,10 @@ static int doubleConsonant(const char *z){
 */
 static int star_oh(const char *z){
   return
-    z[0]!=0 && isConsonant(z) &&
+    isConsonant(z) &&
     z[0]!='w' && z[0]!='x' && z[0]!='y' &&
-    z[1]!=0 && isVowel(z+1) &&
-    z[2]!=0 && isConsonant(z+2);
+    isVowel(z+1) &&
+    isConsonant(z+2);
 }
 
 /*
@@ -293,7 +296,7 @@ static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
   int i, mx, j;
   int hasDigit = 0;
   for(i=0; i<nIn; i++){
-    int c = zIn[i];
+    char c = zIn[i];
     if( c>='A' && c<='Z' ){
       zOut[i] = c - 'A' + 'a';
     }else{
@@ -337,7 +340,7 @@ static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
 ** no chance of overflowing the zOut buffer.
 */
 static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
-  int i, j, c;
+  int i, j;
   char zReverse[28];
   char *z, *z2;
   if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
@@ -347,7 +350,7 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
     return;
   }
   for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
-    c = zIn[i];
+    char c = zIn[i];
     if( c>='A' && c<='Z' ){
       zReverse[j] = c + 'a' - 'A';
     }else if( c>='a' && c<='z' ){
@@ -546,7 +549,7 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
   /* z[] is now the stemmed word in reverse order.  Flip it back
   ** around into forward order and return.
   */
-  *pnOut = i = strlen(z);
+  *pnOut = i = (int)strlen(z);
   zOut[i] = 0;
   while( *z ){
     zOut[--i] = *(z++);
@@ -601,9 +604,11 @@ static int porterNext(
     if( c->iOffset>iStartOffset ){
       int n = c->iOffset-iStartOffset;
       if( n>c->nAllocated ){
+        char *pNew;
         c->nAllocated = n+20;
-        c->zToken = sqlite3_realloc(c->zToken, c->nAllocated);
-        if( c->zToken==NULL ) return SQLITE_NOMEM;
+        pNew = sqlite3_realloc(c->zToken, c->nAllocated);
+        if( !pNew ) return SQLITE_NOMEM;
+        c->zToken = pNew;
       }
       porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
       *pzToken = c->zToken;
diff --git a/third_party/sqlite/src/ext/fts3/fts3_snippet.c b/third_party/sqlite/src/ext/fts3/fts3_snippet.c
new file mode 100644
index 0000000..d67f7ac
--- /dev/null
+++ b/third_party/sqlite/src/ext/fts3/fts3_snippet.c
@@ -0,0 +1,1205 @@
+/*
+** 2009 Oct 23
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+*/
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#include "fts3Int.h"
+#include <string.h>
+#include <assert.h>
+
+
+/*
+** Used as an fts3ExprIterate() context when loading phrase doclists to
+** Fts3Expr.aDoclist[]/nDoclist.
+*/
+typedef struct LoadDoclistCtx LoadDoclistCtx;
+struct LoadDoclistCtx {
+  Fts3Table *pTab;                /* FTS3 Table */
+  int nPhrase;                    /* Number of phrases seen so far */
+  int nToken;                     /* Number of tokens seen so far */
+};
+
+/*
+** The following types are used as part of the implementation of the 
+** fts3BestSnippet() routine.
+*/
+typedef struct SnippetIter SnippetIter;
+typedef struct SnippetPhrase SnippetPhrase;
+typedef struct SnippetFragment SnippetFragment;
+
+struct SnippetIter {
+  Fts3Cursor *pCsr;               /* Cursor snippet is being generated from */
+  int iCol;                       /* Extract snippet from this column */
+  int nSnippet;                   /* Requested snippet length (in tokens) */
+  int nPhrase;                    /* Number of phrases in query */
+  SnippetPhrase *aPhrase;         /* Array of size nPhrase */
+  int iCurrent;                   /* First token of current snippet */
+};
+
+struct SnippetPhrase {
+  int nToken;                     /* Number of tokens in phrase */
+  char *pList;                    /* Pointer to start of phrase position list */
+  int iHead;                      /* Next value in position list */
+  char *pHead;                    /* Position list data following iHead */
+  int iTail;                      /* Next value in trailing position list */
+  char *pTail;                    /* Position list data following iTail */
+};
+
+struct SnippetFragment {
+  int iCol;                       /* Column snippet is extracted from */
+  int iPos;                       /* Index of first token in snippet */
+  u64 covered;                    /* Mask of query phrases covered */
+  u64 hlmask;                     /* Mask of snippet terms to highlight */
+};
+
+/*
+** This type is used as an fts3ExprIterate() context object while 
+** accumulating the data returned by the matchinfo() function.
+*/
+typedef struct MatchInfo MatchInfo;
+struct MatchInfo {
+  Fts3Cursor *pCursor;            /* FTS3 Cursor */
+  int nCol;                       /* Number of columns in table */
+  u32 *aMatchinfo;                /* Pre-allocated buffer */
+};
+
+
+
+/*
+** The snippet() and offsets() functions both return text values. An instance
+** of the following structure is used to accumulate those values while the
+** functions are running. See fts3StringAppend() for details.
+*/
+typedef struct StrBuffer StrBuffer;
+struct StrBuffer {
+  char *z;                        /* Pointer to buffer containing string */
+  int n;                          /* Length of z in bytes (excl. nul-term) */
+  int nAlloc;                     /* Allocated size of buffer z in bytes */
+};
+
+
+/*
+** This function is used to help iterate through a position-list. A position
+** list is a list of unique integers, sorted from smallest to largest. Each
+** element of the list is represented by an FTS3 varint that takes the value
+** of the difference between the current element and the previous one plus
+** two. For example, to store the position-list:
+**
+**     4 9 113
+**
+** the three varints:
+**
+**     6 7 106
+**
+** are encoded.
+**
+** When this function is called, *pp points to the start of an element of
+** the list. *piPos contains the value of the previous entry in the list.
+** After it returns, *piPos contains the value of the next element of the
+** list and *pp is advanced to the following varint.
+*/
+static void fts3GetDeltaPosition(char **pp, int *piPos){
+  int iVal;
+  *pp += sqlite3Fts3GetVarint32(*pp, &iVal);
+  *piPos += (iVal-2);
+}
+
+/*
+** Helper function for fts3ExprIterate() (see below).
+*/
+static int fts3ExprIterate2(
+  Fts3Expr *pExpr,                /* Expression to iterate phrases of */
+  int *piPhrase,                  /* Pointer to phrase counter */
+  int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
+  void *pCtx                      /* Second argument to pass to callback */
+){
+  int rc;                         /* Return code */
+  int eType = pExpr->eType;       /* Type of expression node pExpr */
+
+  if( eType!=FTSQUERY_PHRASE ){
+    assert( pExpr->pLeft && pExpr->pRight );
+    rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx);
+    if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){
+      rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx);
+    }
+  }else{
+    rc = x(pExpr, *piPhrase, pCtx);
+    (*piPhrase)++;
+  }
+  return rc;
+}
+
+/*
+** Iterate through all phrase nodes in an FTS3 query, except those that
+** are part of a sub-tree that is the right-hand-side of a NOT operator.
+** For each phrase node found, the supplied callback function is invoked.
+**
+** If the callback function returns anything other than SQLITE_OK, 
+** the iteration is abandoned and the error code returned immediately.
+** Otherwise, SQLITE_OK is returned after a callback has been made for
+** all eligible phrase nodes.
+*/
+static int fts3ExprIterate(
+  Fts3Expr *pExpr,                /* Expression to iterate phrases of */
+  int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
+  void *pCtx                      /* Second argument to pass to callback */
+){
+  int iPhrase = 0;                /* Variable used as the phrase counter */
+  return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
+}
+
+/*
+** The argument to this function is always a phrase node. Its doclist 
+** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes
+** to the left of this one in the query tree have already been loaded.
+**
+** If this phrase node is part of a series of phrase nodes joined by 
+** NEAR operators (and is not the left-most of said series), then elements are
+** removed from the phrases doclist consistent with the NEAR restriction. If
+** required, elements may be removed from the doclists of phrases to the
+** left of this one that are part of the same series of NEAR operator 
+** connected phrases.
+**
+** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
+*/
+static int fts3ExprNearTrim(Fts3Expr *pExpr){
+  int rc = SQLITE_OK;
+  Fts3Expr *pParent = pExpr->pParent;
+
+  assert( pExpr->eType==FTSQUERY_PHRASE );
+  while( rc==SQLITE_OK
+   && pParent 
+   && pParent->eType==FTSQUERY_NEAR 
+   && pParent->pRight==pExpr 
+  ){
+    /* This expression (pExpr) is the right-hand-side of a NEAR operator. 
+    ** Find the expression to the left of the same operator.
+    */
+    int nNear = pParent->nNear;
+    Fts3Expr *pLeft = pParent->pLeft;
+
+    if( pLeft->eType!=FTSQUERY_PHRASE ){
+      assert( pLeft->eType==FTSQUERY_NEAR );
+      assert( pLeft->pRight->eType==FTSQUERY_PHRASE );
+      pLeft = pLeft->pRight;
+    }
+
+    rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear);
+
+    pExpr = pLeft;
+    pParent = pExpr->pParent;
+  }
+
+  return rc;
+}
+
+/*
+** This is an fts3ExprIterate() callback used while loading the doclists
+** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
+** fts3ExprLoadDoclists().
+*/
+static int fts3ExprLoadDoclistsCb1(Fts3Expr *pExpr, int iPhrase, void *ctx){
+  int rc = SQLITE_OK;
+  LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
+
+  UNUSED_PARAMETER(iPhrase);
+
+  p->nPhrase++;
+  p->nToken += pExpr->pPhrase->nToken;
+
+  if( pExpr->isLoaded==0 ){
+    rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr);
+    pExpr->isLoaded = 1;
+    if( rc==SQLITE_OK ){
+      rc = fts3ExprNearTrim(pExpr);
+    }
+  }
+
+  return rc;
+}
+
+/*
+** This is an fts3ExprIterate() callback used while loading the doclists
+** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
+** fts3ExprLoadDoclists().
+*/
+static int fts3ExprLoadDoclistsCb2(Fts3Expr *pExpr, int iPhrase, void *ctx){
+  UNUSED_PARAMETER(iPhrase);
+  UNUSED_PARAMETER(ctx);
+  if( pExpr->aDoclist ){
+    pExpr->pCurrent = pExpr->aDoclist;
+    pExpr->iCurrent = 0;
+    pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent, &pExpr->iCurrent);
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Load the doclists for each phrase in the query associated with FTS3 cursor
+** pCsr. 
+**
+** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable 
+** phrases in the expression (all phrases except those directly or 
+** indirectly descended from the right-hand-side of a NOT operator). If 
+** pnToken is not NULL, then it is set to the number of tokens in all
+** matchable phrases of the expression.
+*/
+static int fts3ExprLoadDoclists(
+  Fts3Cursor *pCsr,               /* Fts3 cursor for current query */
+  int *pnPhrase,                  /* OUT: Number of phrases in query */
+  int *pnToken                    /* OUT: Number of tokens in query */
+){
+  int rc;                         /* Return Code */
+  LoadDoclistCtx sCtx = {0,0,0};  /* Context for fts3ExprIterate() */
+  sCtx.pTab = (Fts3Table *)pCsr->base.pVtab;
+  rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx);
+  if( rc==SQLITE_OK ){
+    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0);
+  }
+  if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
+  if( pnToken ) *pnToken = sCtx.nToken;
+  return rc;
+}
+
+/*
+** Advance the position list iterator specified by the first two 
+** arguments so that it points to the first element with a value greater
+** than or equal to parameter iNext.
+*/
+static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){
+  char *pIter = *ppIter;
+  if( pIter ){
+    int iIter = *piIter;
+
+    while( iIter<iNext ){
+      if( 0==(*pIter & 0xFE) ){
+        iIter = -1;
+        pIter = 0;
+        break;
+      }
+      fts3GetDeltaPosition(&pIter, &iIter);
+    }
+
+    *piIter = iIter;
+    *ppIter = pIter;
+  }
+}
+
+/*
+** Advance the snippet iterator to the next candidate snippet.
+*/
+static int fts3SnippetNextCandidate(SnippetIter *pIter){
+  int i;                          /* Loop counter */
+
+  if( pIter->iCurrent<0 ){
+    /* The SnippetIter object has just been initialized. The first snippet
+    ** candidate always starts at offset 0 (even if this candidate has a
+    ** score of 0.0).
+    */
+    pIter->iCurrent = 0;
+
+    /* Advance the 'head' iterator of each phrase to the first offset that
+    ** is greater than or equal to (iNext+nSnippet).
+    */
+    for(i=0; i<pIter->nPhrase; i++){
+      SnippetPhrase *pPhrase = &pIter->aPhrase[i];
+      fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet);
+    }
+  }else{
+    int iStart;
+    int iEnd = 0x7FFFFFFF;
+
+    for(i=0; i<pIter->nPhrase; i++){
+      SnippetPhrase *pPhrase = &pIter->aPhrase[i];
+      if( pPhrase->pHead && pPhrase->iHead<iEnd ){
+        iEnd = pPhrase->iHead;
+      }
+    }
+    if( iEnd==0x7FFFFFFF ){
+      return 1;
+    }
+
+    pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1;
+    for(i=0; i<pIter->nPhrase; i++){
+      SnippetPhrase *pPhrase = &pIter->aPhrase[i];
+      fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1);
+      fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart);
+    }
+  }
+
+  return 0;
+}
+
+/*
+** Retrieve information about the current candidate snippet of snippet 
+** iterator pIter.
+*/
+static void fts3SnippetDetails(
+  SnippetIter *pIter,             /* Snippet iterator */
+  u64 mCovered,                   /* Bitmask of phrases already covered */
+  int *piToken,                   /* OUT: First token of proposed snippet */
+  int *piScore,                   /* OUT: "Score" for this snippet */
+  u64 *pmCover,                   /* OUT: Bitmask of phrases covered */
+  u64 *pmHighlight                /* OUT: Bitmask of terms to highlight */
+){
+  int iStart = pIter->iCurrent;   /* First token of snippet */
+  int iScore = 0;                 /* Score of this snippet */
+  int i;                          /* Loop counter */
+  u64 mCover = 0;                 /* Mask of phrases covered by this snippet */
+  u64 mHighlight = 0;             /* Mask of tokens to highlight in snippet */
+
+  for(i=0; i<pIter->nPhrase; i++){
+    SnippetPhrase *pPhrase = &pIter->aPhrase[i];
+    if( pPhrase->pTail ){
+      char *pCsr = pPhrase->pTail;
+      int iCsr = pPhrase->iTail;
+
+      while( iCsr<(iStart+pIter->nSnippet) ){
+        int j;
+        u64 mPhrase = (u64)1 << i;
+        u64 mPos = (u64)1 << (iCsr - iStart);
+        assert( iCsr>=iStart );
+        if( (mCover|mCovered)&mPhrase ){
+          iScore++;
+        }else{
+          iScore += 1000;
+        }
+        mCover |= mPhrase;
+
+        for(j=0; j<pPhrase->nToken; j++){
+          mHighlight |= (mPos>>j);
+        }
+
+        if( 0==(*pCsr & 0x0FE) ) break;
+        fts3GetDeltaPosition(&pCsr, &iCsr);
+      }
+    }
+  }
+
+  /* Set the output variables before returning. */
+  *piToken = iStart;
+  *piScore = iScore;
+  *pmCover = mCover;
+  *pmHighlight = mHighlight;
+}
+
+/*
+** This function is an fts3ExprIterate() callback used by fts3BestSnippet().
+** Each invocation populates an element of the SnippetIter.aPhrase[] array.
+*/
+static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
+  SnippetIter *p = (SnippetIter *)ctx;
+  SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
+  char *pCsr;
+
+  pPhrase->nToken = pExpr->pPhrase->nToken;
+
+  pCsr = sqlite3Fts3FindPositions(pExpr, p->pCsr->iPrevId, p->iCol);
+  if( pCsr ){
+    int iFirst = 0;
+    pPhrase->pList = pCsr;
+    fts3GetDeltaPosition(&pCsr, &iFirst);
+    pPhrase->pHead = pCsr;
+    pPhrase->pTail = pCsr;
+    pPhrase->iHead = iFirst;
+    pPhrase->iTail = iFirst;
+  }else{
+    assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 );
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Select the fragment of text consisting of nFragment contiguous tokens 
+** from column iCol that represent the "best" snippet. The best snippet
+** is the snippet with the highest score, where scores are calculated
+** by adding:
+**
+**   (a) +1 point for each occurence of a matchable phrase in the snippet.
+**
+**   (b) +1000 points for the first occurence of each matchable phrase in 
+**       the snippet for which the corresponding mCovered bit is not set.
+**
+** The selected snippet parameters are stored in structure *pFragment before
+** returning. The score of the selected snippet is stored in *piScore
+** before returning.
+*/
+static int fts3BestSnippet(
+  int nSnippet,                   /* Desired snippet length */
+  Fts3Cursor *pCsr,               /* Cursor to create snippet for */
+  int iCol,                       /* Index of column to create snippet from */
+  u64 mCovered,                   /* Mask of phrases already covered */
+  u64 *pmSeen,                    /* IN/OUT: Mask of phrases seen */
+  SnippetFragment *pFragment,     /* OUT: Best snippet found */
+  int *piScore                    /* OUT: Score of snippet pFragment */
+){
+  int rc;                         /* Return Code */
+  int nList;                      /* Number of phrases in expression */
+  SnippetIter sIter;              /* Iterates through snippet candidates */
+  int nByte;                      /* Number of bytes of space to allocate */
+  int iBestScore = -1;            /* Best snippet score found so far */
+  int i;                          /* Loop counter */
+
+  memset(&sIter, 0, sizeof(sIter));
+
+  /* Iterate through the phrases in the expression to count them. The same
+  ** callback makes sure the doclists are loaded for each phrase.
+  */
+  rc = fts3ExprLoadDoclists(pCsr, &nList, 0);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  /* Now that it is known how many phrases there are, allocate and zero
+  ** the required space using malloc().
+  */
+  nByte = sizeof(SnippetPhrase) * nList;
+  sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte);
+  if( !sIter.aPhrase ){
+    return SQLITE_NOMEM;
+  }
+  memset(sIter.aPhrase, 0, nByte);
+
+  /* Initialize the contents of the SnippetIter object. Then iterate through
+  ** the set of phrases in the expression to populate the aPhrase[] array.
+  */
+  sIter.pCsr = pCsr;
+  sIter.iCol = iCol;
+  sIter.nSnippet = nSnippet;
+  sIter.nPhrase = nList;
+  sIter.iCurrent = -1;
+  (void)fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter);
+
+  /* Set the *pmSeen output variable. */
+  for(i=0; i<nList; i++){
+    if( sIter.aPhrase[i].pHead ){
+      *pmSeen |= (u64)1 << i;
+    }
+  }
+
+  /* Loop through all candidate snippets. Store the best snippet in 
+  ** *pFragment. Store its associated 'score' in iBestScore.
+  */
+  pFragment->iCol = iCol;
+  while( !fts3SnippetNextCandidate(&sIter) ){
+    int iPos;
+    int iScore;
+    u64 mCover;
+    u64 mHighlight;
+    fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover, &mHighlight);
+    assert( iScore>=0 );
+    if( iScore>iBestScore ){
+      pFragment->iPos = iPos;
+      pFragment->hlmask = mHighlight;
+      pFragment->covered = mCover;
+      iBestScore = iScore;
+    }
+  }
+
+  sqlite3_free(sIter.aPhrase);
+  *piScore = iBestScore;
+  return SQLITE_OK;
+}
+
+
+/*
+** Append a string to the string-buffer passed as the first argument.
+**
+** If nAppend is negative, then the length of the string zAppend is
+** determined using strlen().
+*/
+static int fts3StringAppend(
+  StrBuffer *pStr,                /* Buffer to append to */
+  const char *zAppend,            /* Pointer to data to append to buffer */
+  int nAppend                     /* Size of zAppend in bytes (or -1) */
+){
+  if( nAppend<0 ){
+    nAppend = (int)strlen(zAppend);
+  }
+
+  /* If there is insufficient space allocated at StrBuffer.z, use realloc()
+  ** to grow the buffer until so that it is big enough to accomadate the
+  ** appended data.
+  */
+  if( pStr->n+nAppend+1>=pStr->nAlloc ){
+    int nAlloc = pStr->nAlloc+nAppend+100;
+    char *zNew = sqlite3_realloc(pStr->z, nAlloc);
+    if( !zNew ){
+      return SQLITE_NOMEM;
+    }
+    pStr->z = zNew;
+    pStr->nAlloc = nAlloc;
+  }
+
+  /* Append the data to the string buffer. */
+  memcpy(&pStr->z[pStr->n], zAppend, nAppend);
+  pStr->n += nAppend;
+  pStr->z[pStr->n] = '\0';
+
+  return SQLITE_OK;
+}
+
+/*
+** The fts3BestSnippet() function often selects snippets that end with a
+** query term. That is, the final term of the snippet is always a term
+** that requires highlighting. For example, if 'X' is a highlighted term
+** and '.' is a non-highlighted term, BestSnippet() may select:
+**
+**     ........X.....X
+**
+** This function "shifts" the beginning of the snippet forward in the 
+** document so that there are approximately the same number of 
+** non-highlighted terms to the right of the final highlighted term as there
+** are to the left of the first highlighted term. For example, to this:
+**
+**     ....X.....X....
+**
+** This is done as part of extracting the snippet text, not when selecting
+** the snippet. Snippet selection is done based on doclists only, so there
+** is no way for fts3BestSnippet() to know whether or not the document 
+** actually contains terms that follow the final highlighted term. 
+*/
+static int fts3SnippetShift(
+  Fts3Table *pTab,                /* FTS3 table snippet comes from */
+  int nSnippet,                   /* Number of tokens desired for snippet */
+  const char *zDoc,               /* Document text to extract snippet from */
+  int nDoc,                       /* Size of buffer zDoc in bytes */
+  int *piPos,                     /* IN/OUT: First token of snippet */
+  u64 *pHlmask                    /* IN/OUT: Mask of tokens to highlight */
+){
+  u64 hlmask = *pHlmask;          /* Local copy of initial highlight-mask */
+
+  if( hlmask ){
+    int nLeft;                    /* Tokens to the left of first highlight */
+    int nRight;                   /* Tokens to the right of last highlight */
+    int nDesired;                 /* Ideal number of tokens to shift forward */
+
+    for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++);
+    for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++);
+    nDesired = (nLeft-nRight)/2;
+
+    /* Ideally, the start of the snippet should be pushed forward in the
+    ** document nDesired tokens. This block checks if there are actually
+    ** nDesired tokens to the right of the snippet. If so, *piPos and
+    ** *pHlMask are updated to shift the snippet nDesired tokens to the
+    ** right. Otherwise, the snippet is shifted by the number of tokens
+    ** available.
+    */
+    if( nDesired>0 ){
+      int nShift;                 /* Number of tokens to shift snippet by */
+      int iCurrent = 0;           /* Token counter */
+      int rc;                     /* Return Code */
+      sqlite3_tokenizer_module *pMod;
+      sqlite3_tokenizer_cursor *pC;
+      pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
+
+      /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired)
+      ** or more tokens in zDoc/nDoc.
+      */
+      rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+      if( rc!=SQLITE_OK ){
+        return rc;
+      }
+      pC->pTokenizer = pTab->pTokenizer;
+      while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){
+        const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3;
+        rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent);
+      }
+      pMod->xClose(pC);
+      if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; }
+
+      nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet;
+      assert( nShift<=nDesired );
+      if( nShift>0 ){
+        *piPos += nShift;
+        *pHlmask = hlmask >> nShift;
+      }
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Extract the snippet text for fragment pFragment from cursor pCsr and
+** append it to string buffer pOut.
+*/
+static int fts3SnippetText(
+  Fts3Cursor *pCsr,               /* FTS3 Cursor */
+  SnippetFragment *pFragment,     /* Snippet to extract */
+  int iFragment,                  /* Fragment number */
+  int isLast,                     /* True for final fragment in snippet */
+  int nSnippet,                   /* Number of tokens in extracted snippet */
+  const char *zOpen,              /* String inserted before highlighted term */
+  const char *zClose,             /* String inserted after highlighted term */
+  const char *zEllipsis,          /* String inserted between snippets */
+  StrBuffer *pOut                 /* Write output here */
+){
+  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+  int rc;                         /* Return code */
+  const char *zDoc;               /* Document text to extract snippet from */
+  int nDoc;                       /* Size of zDoc in bytes */
+  int iCurrent = 0;               /* Current token number of document */
+  int iEnd = 0;                   /* Byte offset of end of current token */
+  int isShiftDone = 0;            /* True after snippet is shifted */
+  int iPos = pFragment->iPos;     /* First token of snippet */
+  u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */
+  int iCol = pFragment->iCol+1;   /* Query column to extract text from */
+  sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */
+  sqlite3_tokenizer_cursor *pC;   /* Tokenizer cursor open on zDoc/nDoc */
+  const char *ZDUMMY;             /* Dummy argument used with tokenizer */
+  int DUMMY1;                     /* Dummy argument used with tokenizer */
+  
+  zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol);
+  if( zDoc==0 ){
+    if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){
+      return SQLITE_NOMEM;
+    }
+    return SQLITE_OK;
+  }
+  nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol);
+
+  /* Open a token cursor on the document. */
+  pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
+  rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+  pC->pTokenizer = pTab->pTokenizer;
+
+  while( rc==SQLITE_OK ){
+    int iBegin;                   /* Offset in zDoc of start of token */
+    int iFin;                     /* Offset in zDoc of end of token */
+    int isHighlight;              /* True for highlighted terms */
+
+    rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent);
+    if( rc!=SQLITE_OK ){
+      if( rc==SQLITE_DONE ){
+        /* Special case - the last token of the snippet is also the last token
+        ** of the column. Append any punctuation that occurred between the end
+        ** of the previous token and the end of the document to the output. 
+        ** Then break out of the loop. */
+        rc = fts3StringAppend(pOut, &zDoc[iEnd], -1);
+      }
+      break;
+    }
+    if( iCurrent<iPos ){ continue; }
+
+    if( !isShiftDone ){
+      int n = nDoc - iBegin;
+      rc = fts3SnippetShift(pTab, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask);
+      isShiftDone = 1;
+
+      /* Now that the shift has been done, check if the initial "..." are
+      ** required. They are required if (a) this is not the first fragment,
+      ** or (b) this fragment does not begin at position 0 of its column. 
+      */
+      if( rc==SQLITE_OK && (iPos>0 || iFragment>0) ){
+        rc = fts3StringAppend(pOut, zEllipsis, -1);
+      }
+      if( rc!=SQLITE_OK || iCurrent<iPos ) continue;
+    }
+
+    if( iCurrent>=(iPos+nSnippet) ){
+      if( isLast ){
+        rc = fts3StringAppend(pOut, zEllipsis, -1);
+      }
+      break;
+    }
+
+    /* Set isHighlight to true if this term should be highlighted. */
+    isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0;
+
+    if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd);
+    if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1);
+    if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin);
+    if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1);
+
+    iEnd = iFin;
+  }
+
+  pMod->xClose(pC);
+  return rc;
+}
+
+
+/*
+** This function is used to count the entries in a column-list (a 
+** delta-encoded list of term offsets within a single column of a single 
+** row). When this function is called, *ppCollist should point to the
+** beginning of the first varint in the column-list (the varint that
+** contains the position of the first matching term in the column data).
+** Before returning, *ppCollist is set to point to the first byte after
+** the last varint in the column-list (either the 0x00 signifying the end
+** of the position-list, or the 0x01 that precedes the column number of
+** the next column in the position-list).
+**
+** The number of elements in the column-list is returned.
+*/
+static int fts3ColumnlistCount(char **ppCollist){
+  char *pEnd = *ppCollist;
+  char c = 0;
+  int nEntry = 0;
+
+  /* A column-list is terminated by either a 0x01 or 0x00. */
+  while( 0xFE & (*pEnd | c) ){
+    c = *pEnd++ & 0x80;
+    if( !c ) nEntry++;
+  }
+
+  *ppCollist = pEnd;
+  return nEntry;
+}
+
+static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){
+  char *pCsr = *pp;
+  while( *pCsr ){
+    int nHit;
+    sqlite3_int64 iCol = 0;
+    if( *pCsr==0x01 ){
+      pCsr++;
+      pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
+    }
+    nHit = fts3ColumnlistCount(&pCsr);
+    assert( nHit>0 );
+    if( isGlobal ){
+      aOut[iCol*3+1]++;
+    }
+    aOut[iCol*3] += nHit;
+  }
+  pCsr++;
+  *pp = pCsr;
+}
+
+/*
+** fts3ExprIterate() callback used to collect the "global" matchinfo stats
+** for a single query. The "global" stats are those elements of the matchinfo
+** array that are constant for all rows returned by the current query.
+*/
+static int fts3ExprGlobalMatchinfoCb(
+  Fts3Expr *pExpr,                /* Phrase expression node */
+  int iPhrase,                    /* Phrase number (numbered from zero) */
+  void *pCtx                      /* Pointer to MatchInfo structure */
+){
+  MatchInfo *p = (MatchInfo *)pCtx;
+  char *pCsr;
+  char *pEnd;
+  const int iStart = 2 + (iPhrase * p->nCol * 3) + 1;
+
+  assert( pExpr->isLoaded );
+
+  /* Fill in the global hit count matrix row for this phrase. */
+  pCsr = pExpr->aDoclist;
+  pEnd = &pExpr->aDoclist[pExpr->nDoclist];
+  while( pCsr<pEnd ){
+    while( *pCsr++ & 0x80 );      /* Skip past docid. */
+    fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 1);
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** fts3ExprIterate() callback used to collect the "local" matchinfo stats
+** for a single query. The "local" stats are those elements of the matchinfo
+** array that are different for each row returned by the query.
+*/
+static int fts3ExprLocalMatchinfoCb(
+  Fts3Expr *pExpr,                /* Phrase expression node */
+  int iPhrase,                    /* Phrase number */
+  void *pCtx                      /* Pointer to MatchInfo structure */
+){
+  MatchInfo *p = (MatchInfo *)pCtx;
+
+  if( pExpr->aDoclist ){
+    char *pCsr;
+    int iStart = 2 + (iPhrase * p->nCol * 3);
+    int i;
+
+    for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
+
+    pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1);
+    if( pCsr ){
+      fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
+    }
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Populate pCsr->aMatchinfo[] with data for the current row. The 
+** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
+*/
+static int fts3GetMatchinfo(Fts3Cursor *pCsr){
+  MatchInfo sInfo;
+  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+  int rc = SQLITE_OK;
+
+  sInfo.pCursor = pCsr;
+  sInfo.nCol = pTab->nColumn;
+
+  if( pCsr->aMatchinfo==0 ){
+    /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
+    ** matchinfo function has been called for this query. In this case 
+    ** allocate the array used to accumulate the matchinfo data and
+    ** initialize those elements that are constant for every row.
+    */
+    int nPhrase;                  /* Number of phrases */
+    int nMatchinfo;               /* Number of u32 elements in match-info */
+
+    /* Load doclists for each phrase in the query. */
+    rc = fts3ExprLoadDoclists(pCsr, &nPhrase, 0);
+    if( rc!=SQLITE_OK ){
+      return rc;
+    }
+    nMatchinfo = 2 + 3*sInfo.nCol*nPhrase;
+    if( pTab->bHasDocsize ){
+      nMatchinfo += 1 + 2*pTab->nColumn;
+    }
+
+    sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo);
+    if( !sInfo.aMatchinfo ){ 
+      return SQLITE_NOMEM;
+    }
+    memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo);
+
+
+    /* First element of match-info is the number of phrases in the query */
+    sInfo.aMatchinfo[0] = nPhrase;
+    sInfo.aMatchinfo[1] = sInfo.nCol;
+    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo);
+    if( pTab->bHasDocsize ){
+      int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
+      rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]);
+    }
+    pCsr->aMatchinfo = sInfo.aMatchinfo;
+    pCsr->isMatchinfoNeeded = 1;
+  }
+
+  sInfo.aMatchinfo = pCsr->aMatchinfo;
+  if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){
+    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo);
+    if( pTab->bHasDocsize ){
+      int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
+      rc = sqlite3Fts3MatchinfoDocsizeLocal(pCsr, &sInfo.aMatchinfo[ofst]);
+    }
+    pCsr->isMatchinfoNeeded = 0;
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Implementation of snippet() function.
+*/
+void sqlite3Fts3Snippet(
+  sqlite3_context *pCtx,          /* SQLite function call context */
+  Fts3Cursor *pCsr,               /* Cursor object */
+  const char *zStart,             /* Snippet start text - "<b>" */
+  const char *zEnd,               /* Snippet end text - "</b>" */
+  const char *zEllipsis,          /* Snippet ellipsis text - "<b>...</b>" */
+  int iCol,                       /* Extract snippet from this column */
+  int nToken                      /* Approximate number of tokens in snippet */
+){
+  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+  int rc = SQLITE_OK;
+  int i;
+  StrBuffer res = {0, 0, 0};
+
+  /* The returned text includes up to four fragments of text extracted from
+  ** the data in the current row. The first iteration of the for(...) loop
+  ** below attempts to locate a single fragment of text nToken tokens in 
+  ** size that contains at least one instance of all phrases in the query
+  ** expression that appear in the current row. If such a fragment of text
+  ** cannot be found, the second iteration of the loop attempts to locate
+  ** a pair of fragments, and so on.
+  */
+  int nSnippet = 0;               /* Number of fragments in this snippet */
+  SnippetFragment aSnippet[4];    /* Maximum of 4 fragments per snippet */
+  int nFToken = -1;               /* Number of tokens in each fragment */
+
+  if( !pCsr->pExpr ){
+    sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC);
+    return;
+  }
+
+  for(nSnippet=1; 1; nSnippet++){
+
+    int iSnip;                    /* Loop counter 0..nSnippet-1 */
+    u64 mCovered = 0;             /* Bitmask of phrases covered by snippet */
+    u64 mSeen = 0;                /* Bitmask of phrases seen by BestSnippet() */
+
+    if( nToken>=0 ){
+      nFToken = (nToken+nSnippet-1) / nSnippet;
+    }else{
+      nFToken = -1 * nToken;
+    }
+
+    for(iSnip=0; iSnip<nSnippet; iSnip++){
+      int iBestScore = -1;        /* Best score of columns checked so far */
+      int iRead;                  /* Used to iterate through columns */
+      SnippetFragment *pFragment = &aSnippet[iSnip];
+
+      memset(pFragment, 0, sizeof(*pFragment));
+
+      /* Loop through all columns of the table being considered for snippets.
+      ** If the iCol argument to this function was negative, this means all
+      ** columns of the FTS3 table. Otherwise, only column iCol is considered.
+      */
+      for(iRead=0; iRead<pTab->nColumn; iRead++){
+        SnippetFragment sF;
+        int iS;
+        if( iCol>=0 && iRead!=iCol ) continue;
+
+        /* Find the best snippet of nFToken tokens in column iRead. */
+        rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS);
+        if( rc!=SQLITE_OK ){
+          goto snippet_out;
+        }
+        if( iS>iBestScore ){
+          *pFragment = sF;
+          iBestScore = iS;
+        }
+      }
+
+      mCovered |= pFragment->covered;
+    }
+
+    /* If all query phrases seen by fts3BestSnippet() are present in at least
+    ** one of the nSnippet snippet fragments, break out of the loop.
+    */
+    assert( (mCovered&mSeen)==mCovered );
+    if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break;
+  }
+
+  assert( nFToken>0 );
+
+  for(i=0; i<nSnippet && rc==SQLITE_OK; i++){
+    rc = fts3SnippetText(pCsr, &aSnippet[i], 
+        i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res
+    );
+  }
+
+ snippet_out:
+  if( rc!=SQLITE_OK ){
+    sqlite3_result_error_code(pCtx, rc);
+    sqlite3_free(res.z);
+  }else{
+    sqlite3_result_text(pCtx, res.z, -1, sqlite3_free);
+  }
+}
+
+
+typedef struct TermOffset TermOffset;
+typedef struct TermOffsetCtx TermOffsetCtx;
+
+struct TermOffset {
+  char *pList;                    /* Position-list */
+  int iPos;                       /* Position just read from pList */
+  int iOff;                       /* Offset of this term from read positions */
+};
+
+struct TermOffsetCtx {
+  int iCol;                       /* Column of table to populate aTerm for */
+  int iTerm;
+  sqlite3_int64 iDocid;
+  TermOffset *aTerm;
+};
+
+/*
+** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets().
+*/
+static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
+  TermOffsetCtx *p = (TermOffsetCtx *)ctx;
+  int nTerm;                      /* Number of tokens in phrase */
+  int iTerm;                      /* For looping through nTerm phrase terms */
+  char *pList;                    /* Pointer to position list for phrase */
+  int iPos = 0;                   /* First position in position-list */
+
+  UNUSED_PARAMETER(iPhrase);
+  pList = sqlite3Fts3FindPositions(pExpr, p->iDocid, p->iCol);
+  nTerm = pExpr->pPhrase->nToken;
+  if( pList ){
+    fts3GetDeltaPosition(&pList, &iPos);
+    assert( iPos>=0 );
+  }
+
+  for(iTerm=0; iTerm<nTerm; iTerm++){
+    TermOffset *pT = &p->aTerm[p->iTerm++];
+    pT->iOff = nTerm-iTerm-1;
+    pT->pList = pList;
+    pT->iPos = iPos;
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Implementation of offsets() function.
+*/
+void sqlite3Fts3Offsets(
+  sqlite3_context *pCtx,          /* SQLite function call context */
+  Fts3Cursor *pCsr                /* Cursor object */
+){
+  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+  sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule;
+  const char *ZDUMMY;             /* Dummy argument used with xNext() */
+  int NDUMMY;                     /* Dummy argument used with xNext() */
+  int rc;                         /* Return Code */
+  int nToken;                     /* Number of tokens in query */
+  int iCol;                       /* Column currently being processed */
+  StrBuffer res = {0, 0, 0};      /* Result string */
+  TermOffsetCtx sCtx;             /* Context for fts3ExprTermOffsetInit() */
+
+  if( !pCsr->pExpr ){
+    sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC);
+    return;
+  }
+
+  memset(&sCtx, 0, sizeof(sCtx));
+  assert( pCsr->isRequireSeek==0 );
+
+  /* Count the number of terms in the query */
+  rc = fts3ExprLoadDoclists(pCsr, 0, &nToken);
+  if( rc!=SQLITE_OK ) goto offsets_out;
+
+  /* Allocate the array of TermOffset iterators. */
+  sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken);
+  if( 0==sCtx.aTerm ){
+    rc = SQLITE_NOMEM;
+    goto offsets_out;
+  }
+  sCtx.iDocid = pCsr->iPrevId;
+
+  /* Loop through the table columns, appending offset information to 
+  ** string-buffer res for each column.
+  */
+  for(iCol=0; iCol<pTab->nColumn; iCol++){
+    sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */
+    int iStart;
+    int iEnd;
+    int iCurrent;
+    const char *zDoc;
+    int nDoc;
+
+    /* Initialize the contents of sCtx.aTerm[] for column iCol. There is 
+    ** no way that this operation can fail, so the return code from
+    ** fts3ExprIterate() can be discarded.
+    */
+    sCtx.iCol = iCol;
+    sCtx.iTerm = 0;
+    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
+
+    /* Retreive the text stored in column iCol. If an SQL NULL is stored 
+    ** in column iCol, jump immediately to the next iteration of the loop.
+    ** If an OOM occurs while retrieving the data (this can happen if SQLite
+    ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM 
+    ** to the caller. 
+    */
+    zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1);
+    nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
+    if( zDoc==0 ){
+      if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){
+        continue;
+      }
+      rc = SQLITE_NOMEM;
+      goto offsets_out;
+    }
+
+    /* Initialize a tokenizer iterator to iterate through column iCol. */
+    rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+    if( rc!=SQLITE_OK ) goto offsets_out;
+    pC->pTokenizer = pTab->pTokenizer;
+
+    rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
+    while( rc==SQLITE_OK ){
+      int i;                      /* Used to loop through terms */
+      int iMinPos = 0x7FFFFFFF;   /* Position of next token */
+      TermOffset *pTerm = 0;      /* TermOffset associated with next token */
+
+      for(i=0; i<nToken; i++){
+        TermOffset *pT = &sCtx.aTerm[i];
+        if( pT->pList && (pT->iPos-pT->iOff)<iMinPos ){
+          iMinPos = pT->iPos-pT->iOff;
+          pTerm = pT;
+        }
+      }
+
+      if( !pTerm ){
+        /* All offsets for this column have been gathered. */
+        break;
+      }else{
+        assert( iCurrent<=iMinPos );
+        if( 0==(0xFE&*pTerm->pList) ){
+          pTerm->pList = 0;
+        }else{
+          fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos);
+        }
+        while( rc==SQLITE_OK && iCurrent<iMinPos ){
+          rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
+        }
+        if( rc==SQLITE_OK ){
+          char aBuffer[64];
+          sqlite3_snprintf(sizeof(aBuffer), aBuffer, 
+              "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
+          );
+          rc = fts3StringAppend(&res, aBuffer, -1);
+        }else if( rc==SQLITE_DONE ){
+          rc = SQLITE_CORRUPT;
+        }
+      }
+    }
+    if( rc==SQLITE_DONE ){
+      rc = SQLITE_OK;
+    }
+
+    pMod->xClose(pC);
+    if( rc!=SQLITE_OK ) goto offsets_out;
+  }
+
+ offsets_out:
+  sqlite3_free(sCtx.aTerm);
+  assert( rc!=SQLITE_DONE );
+  if( rc!=SQLITE_OK ){
+    sqlite3_result_error_code(pCtx,  rc);
+    sqlite3_free(res.z);
+  }else{
+    sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free);
+  }
+  return;
+}
+
+/*
+** Implementation of matchinfo() function.
+*/
+void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){
+  int rc;
+  if( !pCsr->pExpr ){
+    sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
+    return;
+  }
+  rc = fts3GetMatchinfo(pCsr);
+  if( rc!=SQLITE_OK ){
+    sqlite3_result_error_code(pContext, rc);
+  }else{
+    Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab;
+    int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3);
+    if( pTab->bHasDocsize ){
+      n += sizeof(u32)*(1 + 2*pTab->nColumn);
+    }
+    sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
+  }
+}
+
+#endif
diff --git a/third_party/sqlite/src/ext/fts3/fts3_tokenizer.c b/third_party/sqlite/src/ext/fts3/fts3_tokenizer.c
index e5de9c9..1121b00 100644
--- a/third_party/sqlite/src/ext/fts3/fts3_tokenizer.c
+++ b/third_party/sqlite/src/ext/fts3/fts3_tokenizer.c
@@ -30,10 +30,10 @@
   SQLITE_EXTENSION_INIT1
 #endif
 
-#include "fts3_hash.h"
-#include "fts3_tokenizer.h"
+#include "fts3Int.h"
 #include <assert.h>
 #include <stddef.h>
+#include <string.h>
 
 /*
 ** Implementation of the SQL scalar function for accessing the underlying 
@@ -60,14 +60,14 @@ static void scalarFunc(
   int argc,
   sqlite3_value **argv
 ){
-  fts3Hash *pHash;
+  Fts3Hash *pHash;
   void *pPtr = 0;
   const unsigned char *zName;
   int nName;
 
   assert( argc==1 || argc==2 );
 
-  pHash = (fts3Hash *)sqlite3_user_data(context);
+  pHash = (Fts3Hash *)sqlite3_user_data(context);
 
   zName = sqlite3_value_text(argv[0]);
   nName = sqlite3_value_bytes(argv[0])+1;
@@ -98,6 +98,127 @@ static void scalarFunc(
   sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
 }
 
+static int fts3IsIdChar(char c){
+  static const char isFtsIdChar[] = {
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
+      0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
+      0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
+      0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
+  };
+  return (c&0x80 || isFtsIdChar[(int)(c)]);
+}
+
+const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
+  const char *z1;
+  const char *z2 = 0;
+
+  /* Find the start of the next token. */
+  z1 = zStr;
+  while( z2==0 ){
+    char c = *z1;
+    switch( c ){
+      case '\0': return 0;        /* No more tokens here */
+      case '\'':
+      case '"':
+      case '`': {
+        z2 = z1;
+        while( *++z2 && (*z2!=c || *++z2==c) );
+        break;
+      }
+      case '[':
+        z2 = &z1[1];
+        while( *z2 && z2[0]!=']' ) z2++;
+        if( *z2 ) z2++;
+        break;
+
+      default:
+        if( fts3IsIdChar(*z1) ){
+          z2 = &z1[1];
+          while( fts3IsIdChar(*z2) ) z2++;
+        }else{
+          z1++;
+        }
+    }
+  }
+
+  *pn = (int)(z2-z1);
+  return z1;
+}
+
+int sqlite3Fts3InitTokenizer(
+  Fts3Hash *pHash,                /* Tokenizer hash table */
+  const char *zArg,               /* Possible tokenizer specification */
+  sqlite3_tokenizer **ppTok,      /* OUT: Tokenizer (if applicable) */
+  const char **pzTokenizer,       /* OUT: Set to zArg if is tokenizer */
+  char **pzErr                    /* OUT: Set to malloced error message */
+){
+  int rc;
+  char *z = (char *)zArg;
+  int n;
+  char *zCopy;
+  char *zEnd;                     /* Pointer to nul-term of zCopy */
+  sqlite3_tokenizer_module *m;
+
+  if( !z ){
+    zCopy = sqlite3_mprintf("simple");
+  }else{
+    if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){
+      return SQLITE_OK;
+    }
+    zCopy = sqlite3_mprintf("%s", &z[8]);
+    *pzTokenizer = zArg;
+  }
+  if( !zCopy ){
+    return SQLITE_NOMEM;
+  }
+
+  zEnd = &zCopy[strlen(zCopy)];
+
+  z = (char *)sqlite3Fts3NextToken(zCopy, &n);
+  z[n] = '\0';
+  sqlite3Fts3Dequote(z);
+
+  m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, (int)strlen(z)+1);
+  if( !m ){
+    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
+    rc = SQLITE_ERROR;
+  }else{
+    char const **aArg = 0;
+    int iArg = 0;
+    z = &z[n+1];
+    while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
+      int nNew = sizeof(char *)*(iArg+1);
+      char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew);
+      if( !aNew ){
+        sqlite3_free(zCopy);
+        sqlite3_free((void *)aArg);
+        return SQLITE_NOMEM;
+      }
+      aArg = aNew;
+      aArg[iArg++] = z;
+      z[n] = '\0';
+      sqlite3Fts3Dequote(z);
+      z = &z[n+1];
+    }
+    rc = m->xCreate(iArg, aArg, ppTok);
+    assert( rc!=SQLITE_OK || *ppTok );
+    if( rc!=SQLITE_OK ){
+      *pzErr = sqlite3_mprintf("unknown tokenizer");
+    }else{
+      (*ppTok)->pModule = m; 
+    }
+    sqlite3_free((void *)aArg);
+  }
+
+  sqlite3_free(zCopy);
+  return rc;
+}
+
+
 #ifdef SQLITE_TEST
 
 #include <tcl.h>
@@ -134,7 +255,7 @@ static void testFunc(
   int argc,
   sqlite3_value **argv
 ){
-  fts3Hash *pHash;
+  Fts3Hash *pHash;
   sqlite3_tokenizer_module *p;
   sqlite3_tokenizer *pTokenizer = 0;
   sqlite3_tokenizer_cursor *pCsr = 0;
@@ -167,7 +288,7 @@ static void testFunc(
     zArg = (const char *)sqlite3_value_text(argv[1]);
   }
 
-  pHash = (fts3Hash *)sqlite3_user_data(context);
+  pHash = (Fts3Hash *)sqlite3_user_data(context);
   p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
 
   if( !p ){
@@ -258,7 +379,7 @@ int queryTokenizer(
   sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
   if( SQLITE_ROW==sqlite3_step(pStmt) ){
     if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
-      memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
     }
   }
 
@@ -295,6 +416,9 @@ static void intTestFunc(
   const sqlite3_tokenizer_module *p2;
   sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
 
+  UNUSED_PARAMETER(argc);
+  UNUSED_PARAMETER(argv);
+
   /* Test the query function */
   sqlite3Fts3SimpleTokenizerModule(&p1);
   rc = queryTokenizer(db, "simple", &p2);
@@ -336,16 +460,16 @@ static void intTestFunc(
 */
 int sqlite3Fts3InitHashTable(
   sqlite3 *db, 
-  fts3Hash *pHash, 
+  Fts3Hash *pHash, 
   const char *zName
 ){
   int rc = SQLITE_OK;
   void *p = (void *)pHash;
   const int any = SQLITE_ANY;
-  char *zTest = 0;
-  char *zTest2 = 0;
 
 #ifdef SQLITE_TEST
+  char *zTest = 0;
+  char *zTest2 = 0;
   void *pdb = (void *)db;
   zTest = sqlite3_mprintf("%s_test", zName);
   zTest2 = sqlite3_mprintf("%s_internal_test", zName);
@@ -354,18 +478,21 @@ int sqlite3Fts3InitHashTable(
   }
 #endif
 
-  if( rc!=SQLITE_OK
-   || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
-   || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
+  if( SQLITE_OK!=rc
+   || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
+   || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
 #ifdef SQLITE_TEST
-   || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
-   || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
-   || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
+   || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
+   || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
+   || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
 #endif
-  );
+   );
 
+#ifdef SQLITE_TEST
   sqlite3_free(zTest);
   sqlite3_free(zTest2);
+#endif
+
   return rc;
 }
 
diff --git a/third_party/sqlite/src/ext/fts3/fts3_tokenizer.h b/third_party/sqlite/src/ext/fts3/fts3_tokenizer.h
index 906303d..6156445 100644
--- a/third_party/sqlite/src/ext/fts3/fts3_tokenizer.h
+++ b/third_party/sqlite/src/ext/fts3/fts3_tokenizer.h
@@ -145,4 +145,8 @@ struct sqlite3_tokenizer_cursor {
   /* Tokenizer implementations will typically add additional fields */
 };
 
+int fts3_global_term_cnt(int iTerm, int iCol);
+int fts3_term_cnt(int iTerm, int iCol);
+
+
 #endif /* _FTS3_TOKENIZER_H_ */
diff --git a/third_party/sqlite/src/ext/fts3/fts3_tokenizer1.c b/third_party/sqlite/src/ext/fts3/fts3_tokenizer1.c
index bbc5696..432c35d 100644
--- a/third_party/sqlite/src/ext/fts3/fts3_tokenizer1.c
+++ b/third_party/sqlite/src/ext/fts3/fts3_tokenizer1.c
@@ -24,6 +24,7 @@
 */
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
 
+#include "fts3Int.h"
 
 #include <assert.h>
 #include <stdlib.h>
@@ -48,9 +49,6 @@ typedef struct simple_tokenizer_cursor {
 } simple_tokenizer_cursor;
 
 
-/* Forward declaration */
-static const sqlite3_tokenizer_module simpleTokenizerModule;
-
 static int simpleDelim(simple_tokenizer *t, unsigned char c){
   return c<0x80 && t->delim[c];
 }
@@ -77,7 +75,7 @@ static int simpleCreate(
   ** information on the initial create.
   */
   if( argc>1 ){
-    int i, n = strlen(argv[1]);
+    int i, n = (int)strlen(argv[1]);
     for(i=0; i<n; i++){
       unsigned char ch = argv[1][i];
       /* We explicitly don't support UTF-8 delimiters for now. */
@@ -91,7 +89,7 @@ static int simpleCreate(
     /* Mark non-alphanumeric ASCII characters as delimiters */
     int i;
     for(i=1; i<0x80; i++){
-      t->delim[i] = !fts3_isalnum(i);
+      t->delim[i] = !fts3_isalnum(i) ? -1 : 0;
     }
   }
 
@@ -120,6 +118,8 @@ static int simpleOpen(
 ){
   simple_tokenizer_cursor *c;
 
+  UNUSED_PARAMETER(pTokenizer);
+
   c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
   if( c==NULL ) return SQLITE_NOMEM;
 
@@ -184,16 +184,18 @@ static int simpleNext(
     if( c->iOffset>iStartOffset ){
       int i, n = c->iOffset-iStartOffset;
       if( n>c->nTokenAllocated ){
+        char *pNew;
         c->nTokenAllocated = n+20;
-        c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
-        if( c->pToken==NULL ) return SQLITE_NOMEM;
+        pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated);
+        if( !pNew ) return SQLITE_NOMEM;
+        c->pToken = pNew;
       }
       for(i=0; i<n; i++){
         /* TODO(shess) This needs expansion to handle UTF-8
         ** case-insensitivity.
         */
         unsigned char ch = p[iStartOffset+i];
-        c->pToken[i] = (ch>='A' && ch<='Z') ? ch-'A'+'a' : ch;
+        c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch);
       }
       *ppToken = c->pToken;
       *pnBytes = n;
diff --git a/third_party/sqlite/src/ext/fts3/fts3_write.c b/third_party/sqlite/src/ext/fts3/fts3_write.c
new file mode 100644
index 0000000..e434360
--- /dev/null
+++ b/third_party/sqlite/src/ext/fts3/fts3_write.c
@@ -0,0 +1,2622 @@
+/*
+** 2009 Oct 23
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file is part of the SQLite FTS3 extension module. Specifically,
+** this file contains code to insert, update and delete rows from FTS3
+** tables. It also contains code to merge FTS3 b-tree segments. Some
+** of the sub-routines used to merge segments are also used by the query 
+** code in fts3.c.
+*/
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#include "fts3Int.h"
+#include <string.h>
+#include <assert.h>
+#include <stdlib.h>
+
+typedef struct PendingList PendingList;
+typedef struct SegmentNode SegmentNode;
+typedef struct SegmentWriter SegmentWriter;
+
+/*
+** Data structure used while accumulating terms in the pending-terms hash
+** table. The hash table entry maps from term (a string) to a malloc'd
+** instance of this structure.
+*/
+struct PendingList {
+  int nData;
+  char *aData;
+  int nSpace;
+  sqlite3_int64 iLastDocid;
+  sqlite3_int64 iLastCol;
+  sqlite3_int64 iLastPos;
+};
+
+/*
+** An instance of this structure is used to iterate through the terms on
+** a contiguous set of segment b-tree leaf nodes. Although the details of
+** this structure are only manipulated by code in this file, opaque handles
+** of type Fts3SegReader* are also used by code in fts3.c to iterate through
+** terms when querying the full-text index. See functions:
+**
+**   sqlite3Fts3SegReaderNew()
+**   sqlite3Fts3SegReaderFree()
+**   sqlite3Fts3SegReaderIterate()
+**
+** Methods used to manipulate Fts3SegReader structures:
+**
+**   fts3SegReaderNext()
+**   fts3SegReaderFirstDocid()
+**   fts3SegReaderNextDocid()
+*/
+struct Fts3SegReader {
+  int iIdx;                       /* Index within level, or 0x7FFFFFFF for PT */
+  sqlite3_int64 iStartBlock;
+  sqlite3_int64 iEndBlock;
+  sqlite3_stmt *pStmt;            /* SQL Statement to access leaf nodes */
+  char *aNode;                    /* Pointer to node data (or NULL) */
+  int nNode;                      /* Size of buffer at aNode (or 0) */
+  int nTermAlloc;                 /* Allocated size of zTerm buffer */
+  Fts3HashElem **ppNextElem;
+
+  /* Variables set by fts3SegReaderNext(). These may be read directly
+  ** by the caller. They are valid from the time SegmentReaderNew() returns
+  ** until SegmentReaderNext() returns something other than SQLITE_OK
+  ** (i.e. SQLITE_DONE).
+  */
+  int nTerm;                      /* Number of bytes in current term */
+  char *zTerm;                    /* Pointer to current term */
+  char *aDoclist;                 /* Pointer to doclist of current entry */
+  int nDoclist;                   /* Size of doclist in current entry */
+
+  /* The following variables are used to iterate through the current doclist */
+  char *pOffsetList;
+  sqlite3_int64 iDocid;
+};
+
+#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
+
+/*
+** An instance of this structure is used to create a segment b-tree in the
+** database. The internal details of this type are only accessed by the
+** following functions:
+**
+**   fts3SegWriterAdd()
+**   fts3SegWriterFlush()
+**   fts3SegWriterFree()
+*/
+struct SegmentWriter {
+  SegmentNode *pTree;             /* Pointer to interior tree structure */
+  sqlite3_int64 iFirst;           /* First slot in %_segments written */
+  sqlite3_int64 iFree;            /* Next free slot in %_segments */
+  char *zTerm;                    /* Pointer to previous term buffer */
+  int nTerm;                      /* Number of bytes in zTerm */
+  int nMalloc;                    /* Size of malloc'd buffer at zMalloc */
+  char *zMalloc;                  /* Malloc'd space (possibly) used for zTerm */
+  int nSize;                      /* Size of allocation at aData */
+  int nData;                      /* Bytes of data in aData */
+  char *aData;                    /* Pointer to block from malloc() */
+};
+
+/*
+** Type SegmentNode is used by the following three functions to create
+** the interior part of the segment b+-tree structures (everything except
+** the leaf nodes). These functions and type are only ever used by code
+** within the fts3SegWriterXXX() family of functions described above.
+**
+**   fts3NodeAddTerm()
+**   fts3NodeWrite()
+**   fts3NodeFree()
+*/
+struct SegmentNode {
+  SegmentNode *pParent;           /* Parent node (or NULL for root node) */
+  SegmentNode *pRight;            /* Pointer to right-sibling */
+  SegmentNode *pLeftmost;         /* Pointer to left-most node of this depth */
+  int nEntry;                     /* Number of terms written to node so far */
+  char *zTerm;                    /* Pointer to previous term buffer */
+  int nTerm;                      /* Number of bytes in zTerm */
+  int nMalloc;                    /* Size of malloc'd buffer at zMalloc */
+  char *zMalloc;                  /* Malloc'd space (possibly) used for zTerm */
+  int nData;                      /* Bytes of valid data so far */
+  char *aData;                    /* Node data */
+};
+
+/*
+** Valid values for the second argument to fts3SqlStmt().
+*/
+#define SQL_DELETE_CONTENT             0
+#define SQL_IS_EMPTY                   1
+#define SQL_DELETE_ALL_CONTENT         2 
+#define SQL_DELETE_ALL_SEGMENTS        3
+#define SQL_DELETE_ALL_SEGDIR          4
+#define SQL_DELETE_ALL_DOCSIZE         5
+#define SQL_DELETE_ALL_STAT            6
+#define SQL_SELECT_CONTENT_BY_ROWID    7
+#define SQL_NEXT_SEGMENT_INDEX         8
+#define SQL_INSERT_SEGMENTS            9
+#define SQL_NEXT_SEGMENTS_ID          10
+#define SQL_INSERT_SEGDIR             11
+#define SQL_SELECT_LEVEL              12
+#define SQL_SELECT_ALL_LEVEL          13
+#define SQL_SELECT_LEVEL_COUNT        14
+#define SQL_SELECT_SEGDIR_COUNT_MAX   15
+#define SQL_DELETE_SEGDIR_BY_LEVEL    16
+#define SQL_DELETE_SEGMENTS_RANGE     17
+#define SQL_CONTENT_INSERT            18
+#define SQL_GET_BLOCK                 19
+#define SQL_DELETE_DOCSIZE            20
+#define SQL_REPLACE_DOCSIZE           21
+#define SQL_SELECT_DOCSIZE            22
+#define SQL_SELECT_DOCTOTAL           23
+#define SQL_REPLACE_DOCTOTAL          24
+
+/*
+** This function is used to obtain an SQLite prepared statement handle
+** for the statement identified by the second argument. If successful,
+** *pp is set to the requested statement handle and SQLITE_OK returned.
+** Otherwise, an SQLite error code is returned and *pp is set to 0.
+**
+** If argument apVal is not NULL, then it must point to an array with
+** at least as many entries as the requested statement has bound 
+** parameters. The values are bound to the statements parameters before
+** returning.
+*/
+static int fts3SqlStmt(
+  Fts3Table *p,                   /* Virtual table handle */
+  int eStmt,                      /* One of the SQL_XXX constants above */
+  sqlite3_stmt **pp,              /* OUT: Statement handle */
+  sqlite3_value **apVal           /* Values to bind to statement */
+){
+  const char *azSql[] = {
+/* 0  */  "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
+/* 1  */  "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
+/* 2  */  "DELETE FROM %Q.'%q_content'",
+/* 3  */  "DELETE FROM %Q.'%q_segments'",
+/* 4  */  "DELETE FROM %Q.'%q_segdir'",
+/* 5  */  "DELETE FROM %Q.'%q_docsize'",
+/* 6  */  "DELETE FROM %Q.'%q_stat'",
+/* 7  */  "SELECT * FROM %Q.'%q_content' WHERE rowid=?",
+/* 8  */  "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1",
+/* 9  */  "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
+/* 10 */  "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
+/* 11 */  "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
+
+          /* Return segments in order from oldest to newest.*/ 
+/* 12 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
+            "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
+/* 13 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
+            "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
+
+/* 14 */  "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
+/* 15 */  "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
+
+/* 16 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
+/* 17 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
+/* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%z)",
+/* 19 */  "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",
+/* 20 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
+/* 21 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
+/* 22 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
+/* 23 */  "SELECT value FROM %Q.'%q_stat' WHERE id=0",
+/* 24 */  "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
+  };
+  int rc = SQLITE_OK;
+  sqlite3_stmt *pStmt;
+
+  assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
+  assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
+  
+  pStmt = p->aStmt[eStmt];
+  if( !pStmt ){
+    char *zSql;
+    if( eStmt==SQL_CONTENT_INSERT ){
+      int i;                      /* Iterator variable */  
+      char *zVarlist;             /* The "?, ?, ..." string */
+      zVarlist = (char *)sqlite3_malloc(2*p->nColumn+2);
+      if( !zVarlist ){
+        *pp = 0;
+        return SQLITE_NOMEM;
+      }
+      zVarlist[0] = '?';
+      zVarlist[p->nColumn*2+1] = '\0';
+      for(i=1; i<=p->nColumn; i++){
+        zVarlist[i*2-1] = ',';
+        zVarlist[i*2] = '?';
+      }
+      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, zVarlist);
+    }else{
+      zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
+    }
+    if( !zSql ){
+      rc = SQLITE_NOMEM;
+    }else{
+      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
+      sqlite3_free(zSql);
+      assert( rc==SQLITE_OK || pStmt==0 );
+      p->aStmt[eStmt] = pStmt;
+    }
+  }
+  if( apVal ){
+    int i;
+    int nParam = sqlite3_bind_parameter_count(pStmt);
+    for(i=0; rc==SQLITE_OK && i<nParam; i++){
+      rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
+    }
+  }
+  *pp = pStmt;
+  return rc;
+}
+
+/*
+** Similar to fts3SqlStmt(). Except, after binding the parameters in
+** array apVal[] to the SQL statement identified by eStmt, the statement
+** is executed.
+**
+** Returns SQLITE_OK if the statement is successfully executed, or an
+** SQLite error code otherwise.
+*/
+static void fts3SqlExec(
+  int *pRC,                /* Result code */
+  Fts3Table *p,            /* The FTS3 table */
+  int eStmt,               /* Index of statement to evaluate */
+  sqlite3_value **apVal    /* Parameters to bind */
+){
+  sqlite3_stmt *pStmt;
+  int rc;
+  if( *pRC ) return;
+  rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); 
+  if( rc==SQLITE_OK ){
+    sqlite3_step(pStmt);
+    rc = sqlite3_reset(pStmt);
+  }
+  *pRC = rc;
+}
+
+
+/*
+** Read a single block from the %_segments table. If the specified block
+** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO 
+** etc.) occurs, return the appropriate SQLite error code.
+**
+** Otherwise, if successful, set *pzBlock to point to a buffer containing
+** the block read from the database, and *pnBlock to the size of the read
+** block in bytes.
+**
+** WARNING: The returned buffer is only valid until the next call to 
+** sqlite3Fts3ReadBlock().
+*/
+int sqlite3Fts3ReadBlock(
+  Fts3Table *p,
+  sqlite3_int64 iBlock,
+  char const **pzBlock,
+  int *pnBlock
+){
+  sqlite3_stmt *pStmt;
+  int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
+  if( rc!=SQLITE_OK ) return rc;
+  sqlite3_reset(pStmt);
+
+  if( pzBlock ){
+    sqlite3_bind_int64(pStmt, 1, iBlock);
+    rc = sqlite3_step(pStmt); 
+    if( rc!=SQLITE_ROW ){
+      return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc);
+    }
+  
+    *pnBlock = sqlite3_column_bytes(pStmt, 0);
+    *pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
+    if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
+      return SQLITE_CORRUPT;
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This function ensures that the caller has obtained a shared-cache
+** table-lock on the %_content table. This is required before reading
+** data from the fts3 table. If this lock is not acquired first, then
+** the caller may end up holding read-locks on the %_segments and %_segdir
+** tables, but no read-lock on the %_content table. If this happens 
+** a second connection will be able to write to the fts3 table, but
+** attempting to commit those writes might return SQLITE_LOCKED or
+** SQLITE_LOCKED_SHAREDCACHE (because the commit attempts to obtain 
+** write-locks on the %_segments and %_segdir ** tables). 
+**
+** We try to avoid this because if FTS3 returns any error when committing
+** a transaction, the whole transaction will be rolled back. And this is
+** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can
+** still happen if the user reads data directly from the %_segments or
+** %_segdir tables instead of going through FTS3 though.
+*/
+int sqlite3Fts3ReadLock(Fts3Table *p){
+  int rc;                         /* Return code */
+  sqlite3_stmt *pStmt;            /* Statement used to obtain lock */
+
+  rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_null(pStmt, 1);
+    sqlite3_step(pStmt);
+    rc = sqlite3_reset(pStmt);
+  }
+  return rc;
+}
+
+/*
+** Set *ppStmt to a statement handle that may be used to iterate through
+** all rows in the %_segdir table, from oldest to newest. If successful,
+** return SQLITE_OK. If an error occurs while preparing the statement, 
+** return an SQLite error code.
+**
+** There is only ever one instance of this SQL statement compiled for
+** each FTS3 table.
+**
+** The statement returns the following columns from the %_segdir table:
+**
+**   0: idx
+**   1: start_block
+**   2: leaves_end_block
+**   3: end_block
+**   4: root
+*/
+int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){
+  return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0);
+}
+
+
+/*
+** Append a single varint to a PendingList buffer. SQLITE_OK is returned
+** if successful, or an SQLite error code otherwise.
+**
+** This function also serves to allocate the PendingList structure itself.
+** For example, to create a new PendingList structure containing two
+** varints:
+**
+**   PendingList *p = 0;
+**   fts3PendingListAppendVarint(&p, 1);
+**   fts3PendingListAppendVarint(&p, 2);
+*/
+static int fts3PendingListAppendVarint(
+  PendingList **pp,               /* IN/OUT: Pointer to PendingList struct */
+  sqlite3_int64 i                 /* Value to append to data */
+){
+  PendingList *p = *pp;
+
+  /* Allocate or grow the PendingList as required. */
+  if( !p ){
+    p = sqlite3_malloc(sizeof(*p) + 100);
+    if( !p ){
+      return SQLITE_NOMEM;
+    }
+    p->nSpace = 100;
+    p->aData = (char *)&p[1];
+    p->nData = 0;
+  }
+  else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){
+    int nNew = p->nSpace * 2;
+    p = sqlite3_realloc(p, sizeof(*p) + nNew);
+    if( !p ){
+      sqlite3_free(*pp);
+      *pp = 0;
+      return SQLITE_NOMEM;
+    }
+    p->nSpace = nNew;
+    p->aData = (char *)&p[1];
+  }
+
+  /* Append the new serialized varint to the end of the list. */
+  p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i);
+  p->aData[p->nData] = '\0';
+  *pp = p;
+  return SQLITE_OK;
+}
+
+/*
+** Add a docid/column/position entry to a PendingList structure. Non-zero
+** is returned if the structure is sqlite3_realloced as part of adding
+** the entry. Otherwise, zero.
+**
+** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning.
+** Zero is always returned in this case. Otherwise, if no OOM error occurs,
+** it is set to SQLITE_OK.
+*/
+static int fts3PendingListAppend(
+  PendingList **pp,               /* IN/OUT: PendingList structure */
+  sqlite3_int64 iDocid,           /* Docid for entry to add */
+  sqlite3_int64 iCol,             /* Column for entry to add */
+  sqlite3_int64 iPos,             /* Position of term for entry to add */
+  int *pRc                        /* OUT: Return code */
+){
+  PendingList *p = *pp;
+  int rc = SQLITE_OK;
+
+  assert( !p || p->iLastDocid<=iDocid );
+
+  if( !p || p->iLastDocid!=iDocid ){
+    sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
+    if( p ){
+      assert( p->nData<p->nSpace );
+      assert( p->aData[p->nData]==0 );
+      p->nData++;
+    }
+    if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){
+      goto pendinglistappend_out;
+    }
+    p->iLastCol = -1;
+    p->iLastPos = 0;
+    p->iLastDocid = iDocid;
+  }
+  if( iCol>0 && p->iLastCol!=iCol ){
+    if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1))
+     || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol))
+    ){
+      goto pendinglistappend_out;
+    }
+    p->iLastCol = iCol;
+    p->iLastPos = 0;
+  }
+  if( iCol>=0 ){
+    assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) );
+    rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos);
+    if( rc==SQLITE_OK ){
+      p->iLastPos = iPos;
+    }
+  }
+
+ pendinglistappend_out:
+  *pRc = rc;
+  if( p!=*pp ){
+    *pp = p;
+    return 1;
+  }
+  return 0;
+}
+
+/*
+** Tokenize the nul-terminated string zText and add all tokens to the
+** pending-terms hash-table. The docid used is that currently stored in
+** p->iPrevDocid, and the column is specified by argument iCol.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int fts3PendingTermsAdd(
+  Fts3Table *p,          /* FTS table into which text will be inserted */
+  const char *zText,     /* Text of document to be inseted */
+  int iCol,              /* Column number into which text is inserted */
+  u32 *pnWord            /* OUT: Number of tokens inserted */
+){
+  int rc;
+  int iStart;
+  int iEnd;
+  int iPos;
+  int nWord = 0;
+
+  char const *zToken;
+  int nToken;
+
+  sqlite3_tokenizer *pTokenizer = p->pTokenizer;
+  sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+  sqlite3_tokenizer_cursor *pCsr;
+  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
+      const char**,int*,int*,int*,int*);
+
+  assert( pTokenizer && pModule );
+
+  rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+  pCsr->pTokenizer = pTokenizer;
+
+  xNext = pModule->xNext;
+  while( SQLITE_OK==rc
+      && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
+  ){
+    PendingList *pList;
+ 
+    if( iPos>=nWord ) nWord = iPos+1;
+
+    /* Positions cannot be negative; we use -1 as a terminator internally.
+    ** Tokens must have a non-zero length.
+    */
+    if( iPos<0 || !zToken || nToken<=0 ){
+      rc = SQLITE_ERROR;
+      break;
+    }
+
+    pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken);
+    if( pList ){
+      p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
+    }
+    if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
+      if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){
+        /* Malloc failed while inserting the new entry. This can only 
+        ** happen if there was no previous entry for this token.
+        */
+        assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) );
+        sqlite3_free(pList);
+        rc = SQLITE_NOMEM;
+      }
+    }
+    if( rc==SQLITE_OK ){
+      p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
+    }
+  }
+
+  pModule->xClose(pCsr);
+  *pnWord = nWord;
+  return (rc==SQLITE_DONE ? SQLITE_OK : rc);
+}
+
+/* 
+** Calling this function indicates that subsequent calls to 
+** fts3PendingTermsAdd() are to add term/position-list pairs for the
+** contents of the document with docid iDocid.
+*/
+static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){
+  /* TODO(shess) Explore whether partially flushing the buffer on
+  ** forced-flush would provide better performance.  I suspect that if
+  ** we ordered the doclists by size and flushed the largest until the
+  ** buffer was half empty, that would let the less frequent terms
+  ** generate longer doclists.
+  */
+  if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){
+    int rc = sqlite3Fts3PendingTermsFlush(p);
+    if( rc!=SQLITE_OK ) return rc;
+  }
+  p->iPrevDocid = iDocid;
+  return SQLITE_OK;
+}
+
+void sqlite3Fts3PendingTermsClear(Fts3Table *p){
+  Fts3HashElem *pElem;
+  for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
+    sqlite3_free(fts3HashData(pElem));
+  }
+  fts3HashClear(&p->pendingTerms);
+  p->nPendingData = 0;
+}
+
+/*
+** This function is called by the xUpdate() method as part of an INSERT
+** operation. It adds entries for each term in the new record to the
+** pendingTerms hash table.
+**
+** Argument apVal is the same as the similarly named argument passed to
+** fts3InsertData(). Parameter iDocid is the docid of the new row.
+*/
+static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
+  int i;                          /* Iterator variable */
+  for(i=2; i<p->nColumn+2; i++){
+    const char *zText = (const char *)sqlite3_value_text(apVal[i]);
+    if( zText ){
+      int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
+      if( rc!=SQLITE_OK ){
+        return rc;
+      }
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This function is called by the xUpdate() method for an INSERT operation.
+** The apVal parameter is passed a copy of the apVal argument passed by
+** SQLite to the xUpdate() method. i.e:
+**
+**   apVal[0]                Not used for INSERT.
+**   apVal[1]                rowid
+**   apVal[2]                Left-most user-defined column
+**   ...
+**   apVal[p->nColumn+1]     Right-most user-defined column
+**   apVal[p->nColumn+2]     Hidden column with same name as table
+**   apVal[p->nColumn+3]     Hidden "docid" column (alias for rowid)
+*/
+static int fts3InsertData(
+  Fts3Table *p,                   /* Full-text table */
+  sqlite3_value **apVal,          /* Array of values to insert */
+  sqlite3_int64 *piDocid          /* OUT: Docid for row just inserted */
+){
+  int rc;                         /* Return code */
+  sqlite3_stmt *pContentInsert;   /* INSERT INTO %_content VALUES(...) */
+
+  /* Locate the statement handle used to insert data into the %_content
+  ** table. The SQL for this statement is:
+  **
+  **   INSERT INTO %_content VALUES(?, ?, ?, ...)
+  **
+  ** The statement features N '?' variables, where N is the number of user
+  ** defined columns in the FTS3 table, plus one for the docid field.
+  */
+  rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  /* There is a quirk here. The users INSERT statement may have specified
+  ** a value for the "rowid" field, for the "docid" field, or for both.
+  ** Which is a problem, since "rowid" and "docid" are aliases for the
+  ** same value. For example:
+  **
+  **   INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2);
+  **
+  ** In FTS3, this is an error. It is an error to specify non-NULL values
+  ** for both docid and some other rowid alias.
+  */
+  if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
+    if( SQLITE_NULL==sqlite3_value_type(apVal[0])
+     && SQLITE_NULL!=sqlite3_value_type(apVal[1])
+    ){
+      /* A rowid/docid conflict. */
+      return SQLITE_ERROR;
+    }
+    rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
+    if( rc!=SQLITE_OK ) return rc;
+  }
+
+  /* Execute the statement to insert the record. Set *piDocid to the 
+  ** new docid value. 
+  */
+  sqlite3_step(pContentInsert);
+  rc = sqlite3_reset(pContentInsert);
+
+  *piDocid = sqlite3_last_insert_rowid(p->db);
+  return rc;
+}
+
+
+
+/*
+** Remove all data from the FTS3 table. Clear the hash table containing
+** pending terms.
+*/
+static int fts3DeleteAll(Fts3Table *p){
+  int rc = SQLITE_OK;             /* Return code */
+
+  /* Discard the contents of the pending-terms hash table. */
+  sqlite3Fts3PendingTermsClear(p);
+
+  /* Delete everything from the %_content, %_segments and %_segdir tables. */
+  fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0);
+  fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0);
+  fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
+  if( p->bHasDocsize ){
+    fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0);
+    fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0);
+  }
+  return rc;
+}
+
+/*
+** The first element in the apVal[] array is assumed to contain the docid
+** (an integer) of a row about to be deleted. Remove all terms from the
+** full-text index.
+*/
+static void fts3DeleteTerms(
+  int *pRC,               /* Result code */
+  Fts3Table *p,           /* The FTS table to delete from */
+  sqlite3_value **apVal,  /* apVal[] contains the docid to be deleted */
+  u32 *aSz                /* Sizes of deleted document written here */
+){
+  int rc;
+  sqlite3_stmt *pSelect;
+
+  if( *pRC ) return;
+  rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal);
+  if( rc==SQLITE_OK ){
+    if( SQLITE_ROW==sqlite3_step(pSelect) ){
+      int i;
+      for(i=1; i<=p->nColumn; i++){
+        const char *zText = (const char *)sqlite3_column_text(pSelect, i);
+        rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]);
+        if( rc!=SQLITE_OK ){
+          sqlite3_reset(pSelect);
+          *pRC = rc;
+          return;
+        }
+      }
+    }
+    rc = sqlite3_reset(pSelect);
+  }else{
+    sqlite3_reset(pSelect);
+  }
+  *pRC = rc;
+}
+
+/*
+** Forward declaration to account for the circular dependency between
+** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
+*/
+static int fts3SegmentMerge(Fts3Table *, int);
+
+/* 
+** This function allocates a new level iLevel index in the segdir table.
+** Usually, indexes are allocated within a level sequentially starting
+** with 0, so the allocated index is one greater than the value returned
+** by:
+**
+**   SELECT max(idx) FROM %_segdir WHERE level = :iLevel
+**
+** However, if there are already FTS3_MERGE_COUNT indexes at the requested
+** level, they are merged into a single level (iLevel+1) segment and the 
+** allocated index is 0.
+**
+** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
+** returned. Otherwise, an SQLite error code is returned.
+*/
+static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
+  int rc;                         /* Return Code */
+  sqlite3_stmt *pNextIdx;         /* Query for next idx at level iLevel */
+  int iNext = 0;                  /* Result of query pNextIdx */
+
+  /* Set variable iNext to the next available segdir index at level iLevel. */
+  rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int(pNextIdx, 1, iLevel);
+    if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
+      iNext = sqlite3_column_int(pNextIdx, 0);
+    }
+    rc = sqlite3_reset(pNextIdx);
+  }
+
+  if( rc==SQLITE_OK ){
+    /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
+    ** full, merge all segments in level iLevel into a single iLevel+1
+    ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
+    ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
+    */
+    if( iNext>=FTS3_MERGE_COUNT ){
+      rc = fts3SegmentMerge(p, iLevel);
+      *piIdx = 0;
+    }else{
+      *piIdx = iNext;
+    }
+  }
+
+  return rc;
+}
+
+/*
+** Move the iterator passed as the first argument to the next term in the
+** segment. If successful, SQLITE_OK is returned. If there is no next term,
+** SQLITE_DONE. Otherwise, an SQLite error code.
+*/
+static int fts3SegReaderNext(Fts3SegReader *pReader){
+  char *pNext;                    /* Cursor variable */
+  int nPrefix;                    /* Number of bytes in term prefix */
+  int nSuffix;                    /* Number of bytes in term suffix */
+
+  if( !pReader->aDoclist ){
+    pNext = pReader->aNode;
+  }else{
+    pNext = &pReader->aDoclist[pReader->nDoclist];
+  }
+
+  if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
+    int rc;
+    if( fts3SegReaderIsPending(pReader) ){
+      Fts3HashElem *pElem = *(pReader->ppNextElem);
+      if( pElem==0 ){
+        pReader->aNode = 0;
+      }else{
+        PendingList *pList = (PendingList *)fts3HashData(pElem);
+        pReader->zTerm = (char *)fts3HashKey(pElem);
+        pReader->nTerm = fts3HashKeysize(pElem);
+        pReader->nNode = pReader->nDoclist = pList->nData + 1;
+        pReader->aNode = pReader->aDoclist = pList->aData;
+        pReader->ppNextElem++;
+        assert( pReader->aNode );
+      }
+      return SQLITE_OK;
+    }
+    if( !pReader->pStmt ){
+      pReader->aNode = 0;
+      return SQLITE_OK;
+    }
+    rc = sqlite3_step(pReader->pStmt);
+    if( rc!=SQLITE_ROW ){
+      pReader->aNode = 0;
+      return (rc==SQLITE_DONE ? SQLITE_OK : rc);
+    }
+    pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0);
+    pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0);
+    pNext = pReader->aNode;
+  }
+  
+  pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
+  pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
+
+  if( nPrefix+nSuffix>pReader->nTermAlloc ){
+    int nNew = (nPrefix+nSuffix)*2;
+    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+    if( !zNew ){
+      return SQLITE_NOMEM;
+    }
+    pReader->zTerm = zNew;
+    pReader->nTermAlloc = nNew;
+  }
+  memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
+  pReader->nTerm = nPrefix+nSuffix;
+  pNext += nSuffix;
+  pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
+  assert( pNext<&pReader->aNode[pReader->nNode] );
+  pReader->aDoclist = pNext;
+  pReader->pOffsetList = 0;
+  return SQLITE_OK;
+}
+
+/*
+** Set the SegReader to point to the first docid in the doclist associated
+** with the current term.
+*/
+static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
+  int n;
+  assert( pReader->aDoclist );
+  assert( !pReader->pOffsetList );
+  n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
+  pReader->pOffsetList = &pReader->aDoclist[n];
+}
+
+/*
+** Advance the SegReader to point to the next docid in the doclist
+** associated with the current term.
+** 
+** If arguments ppOffsetList and pnOffsetList are not NULL, then 
+** *ppOffsetList is set to point to the first column-offset list
+** in the doclist entry (i.e. immediately past the docid varint).
+** *pnOffsetList is set to the length of the set of column-offset
+** lists, not including the nul-terminator byte. For example:
+*/
+static void fts3SegReaderNextDocid(
+  Fts3SegReader *pReader,
+  char **ppOffsetList,
+  int *pnOffsetList
+){
+  char *p = pReader->pOffsetList;
+  char c = 0;
+
+  /* Pointer p currently points at the first byte of an offset list. The
+  ** following two lines advance it to point one byte past the end of
+  ** the same offset list.
+  */
+  while( *p | c ) c = *p++ & 0x80;
+  p++;
+
+  /* If required, populate the output variables with a pointer to and the
+  ** size of the previous offset-list.
+  */
+  if( ppOffsetList ){
+    *ppOffsetList = pReader->pOffsetList;
+    *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
+  }
+
+  /* If there are no more entries in the doclist, set pOffsetList to
+  ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
+  ** Fts3SegReader.pOffsetList to point to the next offset list before
+  ** returning.
+  */
+  if( p>=&pReader->aDoclist[pReader->nDoclist] ){
+    pReader->pOffsetList = 0;
+  }else{
+    sqlite3_int64 iDelta;
+    pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
+    pReader->iDocid += iDelta;
+  }
+}
+
+/*
+** Free all allocations associated with the iterator passed as the 
+** second argument.
+*/
+void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){
+  if( pReader ){
+    if( pReader->pStmt ){
+      /* Move the leaf-range SELECT statement to the aLeavesStmt[] array,
+      ** so that it can be reused when required by another query.
+      */
+      assert( p->nLeavesStmt<p->nLeavesTotal );
+      sqlite3_reset(pReader->pStmt);
+      p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
+    }
+    if( !fts3SegReaderIsPending(pReader) ){
+      sqlite3_free(pReader->zTerm);
+    }
+    sqlite3_free(pReader);
+  }
+}
+
+/*
+** Allocate a new SegReader object.
+*/
+int sqlite3Fts3SegReaderNew(
+  Fts3Table *p,                   /* Virtual table handle */
+  int iAge,                       /* Segment "age". */
+  sqlite3_int64 iStartLeaf,       /* First leaf to traverse */
+  sqlite3_int64 iEndLeaf,         /* Final leaf to traverse */
+  sqlite3_int64 iEndBlock,        /* Final block of segment */
+  const char *zRoot,              /* Buffer containing root node */
+  int nRoot,                      /* Size of buffer containing root node */
+  Fts3SegReader **ppReader        /* OUT: Allocated Fts3SegReader */
+){
+  int rc = SQLITE_OK;             /* Return code */
+  Fts3SegReader *pReader;         /* Newly allocated SegReader object */
+  int nExtra = 0;                 /* Bytes to allocate segment root node */
+
+  if( iStartLeaf==0 ){
+    nExtra = nRoot;
+  }
+
+  pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
+  if( !pReader ){
+    return SQLITE_NOMEM;
+  }
+  memset(pReader, 0, sizeof(Fts3SegReader));
+  pReader->iStartBlock = iStartLeaf;
+  pReader->iIdx = iAge;
+  pReader->iEndBlock = iEndBlock;
+
+  if( nExtra ){
+    /* The entire segment is stored in the root node. */
+    pReader->aNode = (char *)&pReader[1];
+    pReader->nNode = nRoot;
+    memcpy(pReader->aNode, zRoot, nRoot);
+  }else{
+    /* If the text of the SQL statement to iterate through a contiguous
+    ** set of entries in the %_segments table has not yet been composed,
+    ** compose it now.
+    */
+    if( !p->zSelectLeaves ){
+      p->zSelectLeaves = sqlite3_mprintf(
+          "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? "
+          "ORDER BY blockid", p->zDb, p->zName
+      );
+      if( !p->zSelectLeaves ){
+        rc = SQLITE_NOMEM;
+        goto finished;
+      }
+    }
+
+    /* If there are no free statements in the aLeavesStmt[] array, prepare
+    ** a new statement now. Otherwise, reuse a prepared statement from
+    ** aLeavesStmt[].
+    */
+    if( p->nLeavesStmt==0 ){
+      if( p->nLeavesTotal==p->nLeavesAlloc ){
+        int nNew = p->nLeavesAlloc + 16;
+        sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc(
+            p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *)
+        );
+        if( !aNew ){
+          rc = SQLITE_NOMEM;
+          goto finished;
+        }
+        p->nLeavesAlloc = nNew;
+        p->aLeavesStmt = aNew;
+      }
+      rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0);
+      if( rc!=SQLITE_OK ){
+        goto finished;
+      }
+      p->nLeavesTotal++;
+    }else{
+      pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt];
+    }
+
+    /* Bind the start and end leaf blockids to the prepared SQL statement. */
+    sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf);
+    sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf);
+  }
+  rc = fts3SegReaderNext(pReader);
+
+ finished:
+  if( rc==SQLITE_OK ){
+    *ppReader = pReader;
+  }else{
+    sqlite3Fts3SegReaderFree(p, pReader);
+  }
+  return rc;
+}
+
+/*
+** This is a comparison function used as a qsort() callback when sorting
+** an array of pending terms by term. This occurs as part of flushing
+** the contents of the pending-terms hash table to the database.
+*/
+static int fts3CompareElemByTerm(const void *lhs, const void *rhs){
+  char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
+  char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
+  int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
+  int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs);
+
+  int n = (n1<n2 ? n1 : n2);
+  int c = memcmp(z1, z2, n);
+  if( c==0 ){
+    c = n1 - n2;
+  }
+  return c;
+}
+
+/*
+** This function is used to allocate an Fts3SegReader that iterates through
+** a subset of the terms stored in the Fts3Table.pendingTerms array.
+*/
+int sqlite3Fts3SegReaderPending(
+  Fts3Table *p,                   /* Virtual table handle */
+  const char *zTerm,              /* Term to search for */
+  int nTerm,                      /* Size of buffer zTerm */
+  int isPrefix,                   /* True for a term-prefix query */
+  Fts3SegReader **ppReader        /* OUT: SegReader for pending-terms */
+){
+  Fts3SegReader *pReader = 0;     /* Fts3SegReader object to return */
+  Fts3HashElem **aElem = 0;       /* Array of term hash entries to scan */
+  int nElem = 0;                  /* Size of array at aElem */
+  int rc = SQLITE_OK;             /* Return Code */
+
+  if( isPrefix ){
+    int nAlloc = 0;               /* Size of allocated array at aElem */
+    Fts3HashElem *pE = 0;         /* Iterator variable */
+
+    for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){
+      char *zKey = (char *)fts3HashKey(pE);
+      int nKey = fts3HashKeysize(pE);
+      if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
+        if( nElem==nAlloc ){
+          Fts3HashElem **aElem2;
+          nAlloc += 16;
+          aElem2 = (Fts3HashElem **)sqlite3_realloc(
+              aElem, nAlloc*sizeof(Fts3HashElem *)
+          );
+          if( !aElem2 ){
+            rc = SQLITE_NOMEM;
+            nElem = 0;
+            break;
+          }
+          aElem = aElem2;
+        }
+        aElem[nElem++] = pE;
+      }
+    }
+
+    /* If more than one term matches the prefix, sort the Fts3HashElem
+    ** objects in term order using qsort(). This uses the same comparison
+    ** callback as is used when flushing terms to disk.
+    */
+    if( nElem>1 ){
+      qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
+    }
+
+  }else{
+    Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm);
+    if( pE ){
+      aElem = &pE;
+      nElem = 1;
+    }
+  }
+
+  if( nElem>0 ){
+    int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *);
+    pReader = (Fts3SegReader *)sqlite3_malloc(nByte);
+    if( !pReader ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memset(pReader, 0, nByte);
+      pReader->iIdx = 0x7FFFFFFF;
+      pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
+      memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
+      fts3SegReaderNext(pReader);
+    }
+  }
+
+  if( isPrefix ){
+    sqlite3_free(aElem);
+  }
+  *ppReader = pReader;
+  return rc;
+}
+
+
+/*
+** The second argument to this function is expected to be a statement of
+** the form:
+**
+**   SELECT 
+**     idx,                  -- col 0
+**     start_block,          -- col 1
+**     leaves_end_block,     -- col 2
+**     end_block,            -- col 3
+**     root                  -- col 4
+**   FROM %_segdir ...
+**
+** This function allocates and initializes a Fts3SegReader structure to
+** iterate through the terms stored in the segment identified by the
+** current row that pStmt is pointing to. 
+**
+** If successful, the Fts3SegReader is left pointing to the first term
+** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error
+** code is returned.
+*/
+static int fts3SegReaderNew(
+  Fts3Table *p,                   /* Virtual table handle */
+  sqlite3_stmt *pStmt,            /* See above */
+  int iAge,                       /* Segment "age". */
+  Fts3SegReader **ppReader        /* OUT: Allocated Fts3SegReader */
+){
+  return sqlite3Fts3SegReaderNew(p, iAge, 
+      sqlite3_column_int64(pStmt, 1),
+      sqlite3_column_int64(pStmt, 2),
+      sqlite3_column_int64(pStmt, 3),
+      sqlite3_column_blob(pStmt, 4),
+      sqlite3_column_bytes(pStmt, 4),
+      ppReader
+  );
+}
+
+/*
+** Compare the entries pointed to by two Fts3SegReader structures. 
+** Comparison is as follows:
+**
+**   1) EOF is greater than not EOF.
+**
+**   2) The current terms (if any) are compared using memcmp(). If one
+**      term is a prefix of another, the longer term is considered the
+**      larger.
+**
+**   3) By segment age. An older segment is considered larger.
+*/
+static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
+  int rc;
+  if( pLhs->aNode && pRhs->aNode ){
+    int rc2 = pLhs->nTerm - pRhs->nTerm;
+    if( rc2<0 ){
+      rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm);
+    }else{
+      rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm);
+    }
+    if( rc==0 ){
+      rc = rc2;
+    }
+  }else{
+    rc = (pLhs->aNode==0) - (pRhs->aNode==0);
+  }
+  if( rc==0 ){
+    rc = pRhs->iIdx - pLhs->iIdx;
+  }
+  assert( rc!=0 );
+  return rc;
+}
+
+/*
+** A different comparison function for SegReader structures. In this
+** version, it is assumed that each SegReader points to an entry in
+** a doclist for identical terms. Comparison is made as follows:
+**
+**   1) EOF (end of doclist in this case) is greater than not EOF.
+**
+**   2) By current docid.
+**
+**   3) By segment age. An older segment is considered larger.
+*/
+static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
+  int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
+  if( rc==0 ){
+    if( pLhs->iDocid==pRhs->iDocid ){
+      rc = pRhs->iIdx - pLhs->iIdx;
+    }else{
+      rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
+    }
+  }
+  assert( pLhs->aNode && pRhs->aNode );
+  return rc;
+}
+
+/*
+** Compare the term that the Fts3SegReader object passed as the first argument
+** points to with the term specified by arguments zTerm and nTerm. 
+**
+** If the pSeg iterator is already at EOF, return 0. Otherwise, return
+** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are
+** equal, or +ve if the pSeg term is greater than zTerm/nTerm.
+*/
+static int fts3SegReaderTermCmp(
+  Fts3SegReader *pSeg,            /* Segment reader object */
+  const char *zTerm,              /* Term to compare to */
+  int nTerm                       /* Size of term zTerm in bytes */
+){
+  int res = 0;
+  if( pSeg->aNode ){
+    if( pSeg->nTerm>nTerm ){
+      res = memcmp(pSeg->zTerm, zTerm, nTerm);
+    }else{
+      res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm);
+    }
+    if( res==0 ){
+      res = pSeg->nTerm-nTerm;
+    }
+  }
+  return res;
+}
+
+/*
+** Argument apSegment is an array of nSegment elements. It is known that
+** the final (nSegment-nSuspect) members are already in sorted order
+** (according to the comparison function provided). This function shuffles
+** the array around until all entries are in sorted order.
+*/
+static void fts3SegReaderSort(
+  Fts3SegReader **apSegment,                     /* Array to sort entries of */
+  int nSegment,                                  /* Size of apSegment array */
+  int nSuspect,                                  /* Unsorted entry count */
+  int (*xCmp)(Fts3SegReader *, Fts3SegReader *)  /* Comparison function */
+){
+  int i;                          /* Iterator variable */
+
+  assert( nSuspect<=nSegment );
+
+  if( nSuspect==nSegment ) nSuspect--;
+  for(i=nSuspect-1; i>=0; i--){
+    int j;
+    for(j=i; j<(nSegment-1); j++){
+      Fts3SegReader *pTmp;
+      if( xCmp(apSegment[j], apSegment[j+1])<0 ) break;
+      pTmp = apSegment[j+1];
+      apSegment[j+1] = apSegment[j];
+      apSegment[j] = pTmp;
+    }
+  }
+
+#ifndef NDEBUG
+  /* Check that the list really is sorted now. */
+  for(i=0; i<(nSuspect-1); i++){
+    assert( xCmp(apSegment[i], apSegment[i+1])<0 );
+  }
+#endif
+}
+
+/* 
+** Insert a record into the %_segments table.
+*/
+static int fts3WriteSegment(
+  Fts3Table *p,                   /* Virtual table handle */
+  sqlite3_int64 iBlock,           /* Block id for new block */
+  char *z,                        /* Pointer to buffer containing block data */
+  int n                           /* Size of buffer z in bytes */
+){
+  sqlite3_stmt *pStmt;
+  int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pStmt, 1, iBlock);
+    sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
+    sqlite3_step(pStmt);
+    rc = sqlite3_reset(pStmt);
+  }
+  return rc;
+}
+
+/* 
+** Insert a record into the %_segdir table.
+*/
+static int fts3WriteSegdir(
+  Fts3Table *p,                   /* Virtual table handle */
+  int iLevel,                     /* Value for "level" field */
+  int iIdx,                       /* Value for "idx" field */
+  sqlite3_int64 iStartBlock,      /* Value for "start_block" field */
+  sqlite3_int64 iLeafEndBlock,    /* Value for "leaves_end_block" field */
+  sqlite3_int64 iEndBlock,        /* Value for "end_block" field */
+  char *zRoot,                    /* Blob value for "root" field */
+  int nRoot                       /* Number of bytes in buffer zRoot */
+){
+  sqlite3_stmt *pStmt;
+  int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int(pStmt, 1, iLevel);
+    sqlite3_bind_int(pStmt, 2, iIdx);
+    sqlite3_bind_int64(pStmt, 3, iStartBlock);
+    sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
+    sqlite3_bind_int64(pStmt, 5, iEndBlock);
+    sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
+    sqlite3_step(pStmt);
+    rc = sqlite3_reset(pStmt);
+  }
+  return rc;
+}
+
+/*
+** Return the size of the common prefix (if any) shared by zPrev and
+** zNext, in bytes. For example, 
+**
+**   fts3PrefixCompress("abc", 3, "abcdef", 6)   // returns 3
+**   fts3PrefixCompress("abX", 3, "abcdef", 6)   // returns 2
+**   fts3PrefixCompress("abX", 3, "Xbcdef", 6)   // returns 0
+*/
+static int fts3PrefixCompress(
+  const char *zPrev,              /* Buffer containing previous term */
+  int nPrev,                      /* Size of buffer zPrev in bytes */
+  const char *zNext,              /* Buffer containing next term */
+  int nNext                       /* Size of buffer zNext in bytes */
+){
+  int n;
+  UNUSED_PARAMETER(nNext);
+  for(n=0; n<nPrev && zPrev[n]==zNext[n]; n++);
+  return n;
+}
+
+/*
+** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger
+** (according to memcmp) than the previous term.
+*/
+static int fts3NodeAddTerm(
+  Fts3Table *p,               /* Virtual table handle */
+  SegmentNode **ppTree,           /* IN/OUT: SegmentNode handle */ 
+  int isCopyTerm,                 /* True if zTerm/nTerm is transient */
+  const char *zTerm,              /* Pointer to buffer containing term */
+  int nTerm                       /* Size of term in bytes */
+){
+  SegmentNode *pTree = *ppTree;
+  int rc;
+  SegmentNode *pNew;
+
+  /* First try to append the term to the current node. Return early if 
+  ** this is possible.
+  */
+  if( pTree ){
+    int nData = pTree->nData;     /* Current size of node in bytes */
+    int nReq = nData;             /* Required space after adding zTerm */
+    int nPrefix;                  /* Number of bytes of prefix compression */
+    int nSuffix;                  /* Suffix length */
+
+    nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm);
+    nSuffix = nTerm-nPrefix;
+
+    nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix;
+    if( nReq<=p->nNodeSize || !pTree->zTerm ){
+
+      if( nReq>p->nNodeSize ){
+        /* An unusual case: this is the first term to be added to the node
+        ** and the static node buffer (p->nNodeSize bytes) is not large
+        ** enough. Use a separately malloced buffer instead This wastes
+        ** p->nNodeSize bytes, but since this scenario only comes about when
+        ** the database contain two terms that share a prefix of almost 2KB, 
+        ** this is not expected to be a serious problem. 
+        */
+        assert( pTree->aData==(char *)&pTree[1] );
+        pTree->aData = (char *)sqlite3_malloc(nReq);
+        if( !pTree->aData ){
+          return SQLITE_NOMEM;
+        }
+      }
+
+      if( pTree->zTerm ){
+        /* There is no prefix-length field for first term in a node */
+        nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix);
+      }
+
+      nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix);
+      memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix);
+      pTree->nData = nData + nSuffix;
+      pTree->nEntry++;
+
+      if( isCopyTerm ){
+        if( pTree->nMalloc<nTerm ){
+          char *zNew = sqlite3_realloc(pTree->zMalloc, nTerm*2);
+          if( !zNew ){
+            return SQLITE_NOMEM;
+          }
+          pTree->nMalloc = nTerm*2;
+          pTree->zMalloc = zNew;
+        }
+        pTree->zTerm = pTree->zMalloc;
+        memcpy(pTree->zTerm, zTerm, nTerm);
+        pTree->nTerm = nTerm;
+      }else{
+        pTree->zTerm = (char *)zTerm;
+        pTree->nTerm = nTerm;
+      }
+      return SQLITE_OK;
+    }
+  }
+
+  /* If control flows to here, it was not possible to append zTerm to the
+  ** current node. Create a new node (a right-sibling of the current node).
+  ** If this is the first node in the tree, the term is added to it.
+  **
+  ** Otherwise, the term is not added to the new node, it is left empty for
+  ** now. Instead, the term is inserted into the parent of pTree. If pTree 
+  ** has no parent, one is created here.
+  */
+  pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize);
+  if( !pNew ){
+    return SQLITE_NOMEM;
+  }
+  memset(pNew, 0, sizeof(SegmentNode));
+  pNew->nData = 1 + FTS3_VARINT_MAX;
+  pNew->aData = (char *)&pNew[1];
+
+  if( pTree ){
+    SegmentNode *pParent = pTree->pParent;
+    rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm);
+    if( pTree->pParent==0 ){
+      pTree->pParent = pParent;
+    }
+    pTree->pRight = pNew;
+    pNew->pLeftmost = pTree->pLeftmost;
+    pNew->pParent = pParent;
+    pNew->zMalloc = pTree->zMalloc;
+    pNew->nMalloc = pTree->nMalloc;
+    pTree->zMalloc = 0;
+  }else{
+    pNew->pLeftmost = pNew;
+    rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); 
+  }
+
+  *ppTree = pNew;
+  return rc;
+}
+
+/*
+** Helper function for fts3NodeWrite().
+*/
+static int fts3TreeFinishNode(
+  SegmentNode *pTree, 
+  int iHeight, 
+  sqlite3_int64 iLeftChild
+){
+  int nStart;
+  assert( iHeight>=1 && iHeight<128 );
+  nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild);
+  pTree->aData[nStart] = (char)iHeight;
+  sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild);
+  return nStart;
+}
+
+/*
+** Write the buffer for the segment node pTree and all of its peers to the
+** database. Then call this function recursively to write the parent of 
+** pTree and its peers to the database. 
+**
+** Except, if pTree is a root node, do not write it to the database. Instead,
+** set output variables *paRoot and *pnRoot to contain the root node.
+**
+** If successful, SQLITE_OK is returned and output variable *piLast is
+** set to the largest blockid written to the database (or zero if no
+** blocks were written to the db). Otherwise, an SQLite error code is 
+** returned.
+*/
+static int fts3NodeWrite(
+  Fts3Table *p,                   /* Virtual table handle */
+  SegmentNode *pTree,             /* SegmentNode handle */
+  int iHeight,                    /* Height of this node in tree */
+  sqlite3_int64 iLeaf,            /* Block id of first leaf node */
+  sqlite3_int64 iFree,            /* Block id of next free slot in %_segments */
+  sqlite3_int64 *piLast,          /* OUT: Block id of last entry written */
+  char **paRoot,                  /* OUT: Data for root node */
+  int *pnRoot                     /* OUT: Size of root node in bytes */
+){
+  int rc = SQLITE_OK;
+
+  if( !pTree->pParent ){
+    /* Root node of the tree. */
+    int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf);
+    *piLast = iFree-1;
+    *pnRoot = pTree->nData - nStart;
+    *paRoot = &pTree->aData[nStart];
+  }else{
+    SegmentNode *pIter;
+    sqlite3_int64 iNextFree = iFree;
+    sqlite3_int64 iNextLeaf = iLeaf;
+    for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){
+      int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf);
+      int nWrite = pIter->nData - nStart;
+  
+      rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite);
+      iNextFree++;
+      iNextLeaf += (pIter->nEntry+1);
+    }
+    if( rc==SQLITE_OK ){
+      assert( iNextLeaf==iFree );
+      rc = fts3NodeWrite(
+          p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot
+      );
+    }
+  }
+
+  return rc;
+}
+
+/*
+** Free all memory allocations associated with the tree pTree.
+*/
+static void fts3NodeFree(SegmentNode *pTree){
+  if( pTree ){
+    SegmentNode *p = pTree->pLeftmost;
+    fts3NodeFree(p->pParent);
+    while( p ){
+      SegmentNode *pRight = p->pRight;
+      if( p->aData!=(char *)&p[1] ){
+        sqlite3_free(p->aData);
+      }
+      assert( pRight==0 || p->zMalloc==0 );
+      sqlite3_free(p->zMalloc);
+      sqlite3_free(p);
+      p = pRight;
+    }
+  }
+}
+
+/*
+** Add a term to the segment being constructed by the SegmentWriter object
+** *ppWriter. When adding the first term to a segment, *ppWriter should
+** be passed NULL. This function will allocate a new SegmentWriter object
+** and return it via the input/output variable *ppWriter in this case.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int fts3SegWriterAdd(
+  Fts3Table *p,                   /* Virtual table handle */
+  SegmentWriter **ppWriter,       /* IN/OUT: SegmentWriter handle */ 
+  int isCopyTerm,                 /* True if buffer zTerm must be copied */
+  const char *zTerm,              /* Pointer to buffer containing term */
+  int nTerm,                      /* Size of term in bytes */
+  const char *aDoclist,           /* Pointer to buffer containing doclist */
+  int nDoclist                    /* Size of doclist in bytes */
+){
+  int nPrefix;                    /* Size of term prefix in bytes */
+  int nSuffix;                    /* Size of term suffix in bytes */
+  int nReq;                       /* Number of bytes required on leaf page */
+  int nData;
+  SegmentWriter *pWriter = *ppWriter;
+
+  if( !pWriter ){
+    int rc;
+    sqlite3_stmt *pStmt;
+
+    /* Allocate the SegmentWriter structure */
+    pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter));
+    if( !pWriter ) return SQLITE_NOMEM;
+    memset(pWriter, 0, sizeof(SegmentWriter));
+    *ppWriter = pWriter;
+
+    /* Allocate a buffer in which to accumulate data */
+    pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize);
+    if( !pWriter->aData ) return SQLITE_NOMEM;
+    pWriter->nSize = p->nNodeSize;
+
+    /* Find the next free blockid in the %_segments table */
+    rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0);
+    if( rc!=SQLITE_OK ) return rc;
+    if( SQLITE_ROW==sqlite3_step(pStmt) ){
+      pWriter->iFree = sqlite3_column_int64(pStmt, 0);
+      pWriter->iFirst = pWriter->iFree;
+    }
+    rc = sqlite3_reset(pStmt);
+    if( rc!=SQLITE_OK ) return rc;
+  }
+  nData = pWriter->nData;
+
+  nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm);
+  nSuffix = nTerm-nPrefix;
+
+  /* Figure out how many bytes are required by this new entry */
+  nReq = sqlite3Fts3VarintLen(nPrefix) +    /* varint containing prefix size */
+    sqlite3Fts3VarintLen(nSuffix) +         /* varint containing suffix size */
+    nSuffix +                               /* Term suffix */
+    sqlite3Fts3VarintLen(nDoclist) +        /* Size of doclist */
+    nDoclist;                               /* Doclist data */
+
+  if( nData>0 && nData+nReq>p->nNodeSize ){
+    int rc;
+
+    /* The current leaf node is full. Write it out to the database. */
+    rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
+    if( rc!=SQLITE_OK ) return rc;
+
+    /* Add the current term to the interior node tree. The term added to
+    ** the interior tree must:
+    **
+    **   a) be greater than the largest term on the leaf node just written
+    **      to the database (still available in pWriter->zTerm), and
+    **
+    **   b) be less than or equal to the term about to be added to the new
+    **      leaf node (zTerm/nTerm).
+    **
+    ** In other words, it must be the prefix of zTerm 1 byte longer than
+    ** the common prefix (if any) of zTerm and pWriter->zTerm.
+    */
+    assert( nPrefix<nTerm );
+    rc = fts3NodeAddTerm(p, &pWriter->pTree, isCopyTerm, zTerm, nPrefix+1);
+    if( rc!=SQLITE_OK ) return rc;
+
+    nData = 0;
+    pWriter->nTerm = 0;
+
+    nPrefix = 0;
+    nSuffix = nTerm;
+    nReq = 1 +                              /* varint containing prefix size */
+      sqlite3Fts3VarintLen(nTerm) +         /* varint containing suffix size */
+      nTerm +                               /* Term suffix */
+      sqlite3Fts3VarintLen(nDoclist) +      /* Size of doclist */
+      nDoclist;                             /* Doclist data */
+  }
+
+  /* If the buffer currently allocated is too small for this entry, realloc
+  ** the buffer to make it large enough.
+  */
+  if( nReq>pWriter->nSize ){
+    char *aNew = sqlite3_realloc(pWriter->aData, nReq);
+    if( !aNew ) return SQLITE_NOMEM;
+    pWriter->aData = aNew;
+    pWriter->nSize = nReq;
+  }
+  assert( nData+nReq<=pWriter->nSize );
+
+  /* Append the prefix-compressed term and doclist to the buffer. */
+  nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix);
+  nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix);
+  memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix);
+  nData += nSuffix;
+  nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist);
+  memcpy(&pWriter->aData[nData], aDoclist, nDoclist);
+  pWriter->nData = nData + nDoclist;
+
+  /* Save the current term so that it can be used to prefix-compress the next.
+  ** If the isCopyTerm parameter is true, then the buffer pointed to by
+  ** zTerm is transient, so take a copy of the term data. Otherwise, just
+  ** store a copy of the pointer.
+  */
+  if( isCopyTerm ){
+    if( nTerm>pWriter->nMalloc ){
+      char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2);
+      if( !zNew ){
+        return SQLITE_NOMEM;
+      }
+      pWriter->nMalloc = nTerm*2;
+      pWriter->zMalloc = zNew;
+      pWriter->zTerm = zNew;
+    }
+    assert( pWriter->zTerm==pWriter->zMalloc );
+    memcpy(pWriter->zTerm, zTerm, nTerm);
+  }else{
+    pWriter->zTerm = (char *)zTerm;
+  }
+  pWriter->nTerm = nTerm;
+
+  return SQLITE_OK;
+}
+
+/*
+** Flush all data associated with the SegmentWriter object pWriter to the
+** database. This function must be called after all terms have been added
+** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is
+** returned. Otherwise, an SQLite error code.
+*/
+static int fts3SegWriterFlush(
+  Fts3Table *p,                   /* Virtual table handle */
+  SegmentWriter *pWriter,         /* SegmentWriter to flush to the db */
+  int iLevel,                     /* Value for 'level' column of %_segdir */
+  int iIdx                        /* Value for 'idx' column of %_segdir */
+){
+  int rc;                         /* Return code */
+  if( pWriter->pTree ){
+    sqlite3_int64 iLast = 0;      /* Largest block id written to database */
+    sqlite3_int64 iLastLeaf;      /* Largest leaf block id written to db */
+    char *zRoot = NULL;           /* Pointer to buffer containing root node */
+    int nRoot = 0;                /* Size of buffer zRoot */
+
+    iLastLeaf = pWriter->iFree;
+    rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData);
+    if( rc==SQLITE_OK ){
+      rc = fts3NodeWrite(p, pWriter->pTree, 1,
+          pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot);
+    }
+    if( rc==SQLITE_OK ){
+      rc = fts3WriteSegdir(
+          p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot);
+    }
+  }else{
+    /* The entire tree fits on the root node. Write it to the segdir table. */
+    rc = fts3WriteSegdir(
+        p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
+  }
+  return rc;
+}
+
+/*
+** Release all memory held by the SegmentWriter object passed as the 
+** first argument.
+*/
+static void fts3SegWriterFree(SegmentWriter *pWriter){
+  if( pWriter ){
+    sqlite3_free(pWriter->aData);
+    sqlite3_free(pWriter->zMalloc);
+    fts3NodeFree(pWriter->pTree);
+    sqlite3_free(pWriter);
+  }
+}
+
+/*
+** The first value in the apVal[] array is assumed to contain an integer.
+** This function tests if there exist any documents with docid values that
+** are different from that integer. i.e. if deleting the document with docid
+** apVal[0] would mean the FTS3 table were empty.
+**
+** If successful, *pisEmpty is set to true if the table is empty except for
+** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an
+** error occurs, an SQLite error code is returned.
+*/
+static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){
+  sqlite3_stmt *pStmt;
+  int rc;
+  rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal);
+  if( rc==SQLITE_OK ){
+    if( SQLITE_ROW==sqlite3_step(pStmt) ){
+      *pisEmpty = sqlite3_column_int(pStmt, 0);
+    }
+    rc = sqlite3_reset(pStmt);
+  }
+  return rc;
+}
+
+/*
+** Set *pnSegment to the number of segments of level iLevel in the database.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if not.
+*/
+static int fts3SegmentCount(Fts3Table *p, int iLevel, int *pnSegment){
+  sqlite3_stmt *pStmt;
+  int rc;
+
+  assert( iLevel>=0 );
+  rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0);
+  if( rc!=SQLITE_OK ) return rc;
+  sqlite3_bind_int(pStmt, 1, iLevel);
+  if( SQLITE_ROW==sqlite3_step(pStmt) ){
+    *pnSegment = sqlite3_column_int(pStmt, 0);
+  }
+  return sqlite3_reset(pStmt);
+}
+
+/*
+** Set *pnSegment to the total number of segments in the database. Set
+** *pnMax to the largest segment level in the database (segment levels
+** are stored in the 'level' column of the %_segdir table).
+**
+** Return SQLITE_OK if successful, or an SQLite error code if not.
+*/
+static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){
+  sqlite3_stmt *pStmt;
+  int rc;
+
+  rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0);
+  if( rc!=SQLITE_OK ) return rc;
+  if( SQLITE_ROW==sqlite3_step(pStmt) ){
+    *pnSegment = sqlite3_column_int(pStmt, 0);
+    *pnMax = sqlite3_column_int(pStmt, 1);
+  }
+  return sqlite3_reset(pStmt);
+}
+
+/*
+** This function is used after merging multiple segments into a single large
+** segment to delete the old, now redundant, segment b-trees. Specifically,
+** it:
+** 
+**   1) Deletes all %_segments entries for the segments associated with 
+**      each of the SegReader objects in the array passed as the third 
+**      argument, and
+**
+**   2) deletes all %_segdir entries with level iLevel, or all %_segdir
+**      entries regardless of level if (iLevel<0).
+**
+** SQLITE_OK is returned if successful, otherwise an SQLite error code.
+*/
+static int fts3DeleteSegdir(
+  Fts3Table *p,                   /* Virtual table handle */
+  int iLevel,                     /* Level of %_segdir entries to delete */
+  Fts3SegReader **apSegment,      /* Array of SegReader objects */
+  int nReader                     /* Size of array apSegment */
+){
+  int rc;                         /* Return Code */
+  int i;                          /* Iterator variable */
+  sqlite3_stmt *pDelete;          /* SQL statement to delete rows */
+
+  rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0);
+  for(i=0; rc==SQLITE_OK && i<nReader; i++){
+    Fts3SegReader *pSegment = apSegment[i];
+    if( pSegment->iStartBlock ){
+      sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock);
+      sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock);
+      sqlite3_step(pDelete);
+      rc = sqlite3_reset(pDelete);
+    }
+  }
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  if( iLevel>=0 ){
+    rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_int(pDelete, 1, iLevel);
+      sqlite3_step(pDelete);
+      rc = sqlite3_reset(pDelete);
+    }
+  }else{
+    fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
+  }
+
+  return rc;
+}
+
+/*
+** When this function is called, buffer *ppList (size *pnList bytes) contains 
+** a position list that may (or may not) feature multiple columns. This
+** function adjusts the pointer *ppList and the length *pnList so that they
+** identify the subset of the position list that corresponds to column iCol.
+**
+** If there are no entries in the input position list for column iCol, then
+** *pnList is set to zero before returning.
+*/
+static void fts3ColumnFilter(
+  int iCol,                       /* Column to filter on */
+  char **ppList,                  /* IN/OUT: Pointer to position list */
+  int *pnList                     /* IN/OUT: Size of buffer *ppList in bytes */
+){
+  char *pList = *ppList;
+  int nList = *pnList;
+  char *pEnd = &pList[nList];
+  int iCurrent = 0;
+  char *p = pList;
+
+  assert( iCol>=0 );
+  while( 1 ){
+    char c = 0;
+    while( p<pEnd && (c | *p)&0xFE ) c = *p++ & 0x80;
+  
+    if( iCol==iCurrent ){
+      nList = (int)(p - pList);
+      break;
+    }
+
+    nList -= (int)(p - pList);
+    pList = p;
+    if( nList==0 ){
+      break;
+    }
+    p = &pList[1];
+    p += sqlite3Fts3GetVarint32(p, &iCurrent);
+  }
+
+  *ppList = pList;
+  *pnList = nList;
+}
+
+/*
+** sqlite3Fts3SegReaderIterate() callback used when merging multiple 
+** segments to create a single, larger segment.
+*/
+static int fts3MergeCallback(
+  Fts3Table *p,                   /* FTS3 Virtual table handle */
+  void *pContext,                 /* Pointer to SegmentWriter* to write with */
+  char *zTerm,                    /* Term to write to the db */
+  int nTerm,                      /* Number of bytes in zTerm */
+  char *aDoclist,                 /* Doclist associated with zTerm */
+  int nDoclist                    /* Number of bytes in doclist */
+){
+  SegmentWriter **ppW = (SegmentWriter **)pContext;
+  return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
+}
+
+/*
+** sqlite3Fts3SegReaderIterate() callback used when flushing the contents
+** of the pending-terms hash table to the database.
+*/
+static int fts3FlushCallback(
+  Fts3Table *p,                   /* FTS3 Virtual table handle */
+  void *pContext,                 /* Pointer to SegmentWriter* to write with */
+  char *zTerm,                    /* Term to write to the db */
+  int nTerm,                      /* Number of bytes in zTerm */
+  char *aDoclist,                 /* Doclist associated with zTerm */
+  int nDoclist                    /* Number of bytes in doclist */
+){
+  SegmentWriter **ppW = (SegmentWriter **)pContext;
+  return fts3SegWriterAdd(p, ppW, 0, zTerm, nTerm, aDoclist, nDoclist);
+}
+
+/*
+** This function is used to iterate through a contiguous set of terms 
+** stored in the full-text index. It merges data contained in one or 
+** more segments to support this.
+**
+** The second argument is passed an array of pointers to SegReader objects
+** allocated with sqlite3Fts3SegReaderNew(). This function merges the range 
+** of terms selected by each SegReader. If a single term is present in
+** more than one segment, the associated doclists are merged. For each
+** term and (possibly merged) doclist in the merged range, the callback
+** function xFunc is invoked with its arguments set as follows.
+**
+**   arg 0: Copy of 'p' parameter passed to this function
+**   arg 1: Copy of 'pContext' parameter passed to this function
+**   arg 2: Pointer to buffer containing term
+**   arg 3: Size of arg 2 buffer in bytes
+**   arg 4: Pointer to buffer containing doclist
+**   arg 5: Size of arg 2 buffer in bytes
+**
+** The 4th argument to this function is a pointer to a structure of type
+** Fts3SegFilter, defined in fts3Int.h. The contents of this structure
+** further restrict the range of terms that callbacks are made for and
+** modify the behaviour of this function. See comments above structure
+** definition for details.
+*/
+int sqlite3Fts3SegReaderIterate(
+  Fts3Table *p,                   /* Virtual table handle */
+  Fts3SegReader **apSegment,      /* Array of Fts3SegReader objects */
+  int nSegment,                   /* Size of apSegment array */
+  Fts3SegFilter *pFilter,         /* Restrictions on range of iteration */
+  int (*xFunc)(Fts3Table *, void *, char *, int, char *, int),  /* Callback */
+  void *pContext                  /* Callback context (2nd argument) */
+){
+  int i;                          /* Iterator variable */
+  char *aBuffer = 0;              /* Buffer to merge doclists in */
+  int nAlloc = 0;                 /* Allocated size of aBuffer buffer */
+  int rc = SQLITE_OK;             /* Return code */
+
+  int isIgnoreEmpty =  (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
+  int isRequirePos =   (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
+  int isColFilter =    (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
+  int isPrefix =       (pFilter->flags & FTS3_SEGMENT_PREFIX);
+
+  /* If there are zero segments, this function is a no-op. This scenario
+  ** comes about only when reading from an empty database.
+  */
+  if( nSegment==0 ) goto finished;
+
+  /* If the Fts3SegFilter defines a specific term (or term prefix) to search 
+  ** for, then advance each segment iterator until it points to a term of
+  ** equal or greater value than the specified term. This prevents many
+  ** unnecessary merge/sort operations for the case where single segment
+  ** b-tree leaf nodes contain more than one term.
+  */
+  if( pFilter->zTerm ){
+    int nTerm = pFilter->nTerm;
+    const char *zTerm = pFilter->zTerm;
+    for(i=0; i<nSegment; i++){
+      Fts3SegReader *pSeg = apSegment[i];
+      while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
+        rc = fts3SegReaderNext(pSeg);
+        if( rc!=SQLITE_OK ) goto finished; }
+    }
+  }
+
+  fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
+  while( apSegment[0]->aNode ){
+    int nTerm = apSegment[0]->nTerm;
+    char *zTerm = apSegment[0]->zTerm;
+    int nMerge = 1;
+
+    /* If this is a prefix-search, and if the term that apSegment[0] points
+    ** to does not share a suffix with pFilter->zTerm/nTerm, then all 
+    ** required callbacks have been made. In this case exit early.
+    **
+    ** Similarly, if this is a search for an exact match, and the first term
+    ** of segment apSegment[0] is not a match, exit early.
+    */
+    if( pFilter->zTerm ){
+      if( nTerm<pFilter->nTerm 
+       || (!isPrefix && nTerm>pFilter->nTerm)
+       || memcmp(zTerm, pFilter->zTerm, pFilter->nTerm) 
+    ){
+        goto finished;
+      }
+    }
+
+    while( nMerge<nSegment 
+        && apSegment[nMerge]->aNode
+        && apSegment[nMerge]->nTerm==nTerm 
+        && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm)
+    ){
+      nMerge++;
+    }
+
+    assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
+    if( nMerge==1 && !isIgnoreEmpty ){
+      Fts3SegReader *p0 = apSegment[0];
+      rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist);
+      if( rc!=SQLITE_OK ) goto finished;
+    }else{
+      int nDoclist = 0;           /* Size of doclist */
+      sqlite3_int64 iPrev = 0;    /* Previous docid stored in doclist */
+
+      /* The current term of the first nMerge entries in the array
+      ** of Fts3SegReader objects is the same. The doclists must be merged
+      ** and a single term added to the new segment.
+      */
+      for(i=0; i<nMerge; i++){
+        fts3SegReaderFirstDocid(apSegment[i]);
+      }
+      fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
+      while( apSegment[0]->pOffsetList ){
+        int j;                    /* Number of segments that share a docid */
+        char *pList;
+        int nList;
+        int nByte;
+        sqlite3_int64 iDocid = apSegment[0]->iDocid;
+        fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
+        j = 1;
+        while( j<nMerge
+            && apSegment[j]->pOffsetList
+            && apSegment[j]->iDocid==iDocid
+        ){
+          fts3SegReaderNextDocid(apSegment[j], 0, 0);
+          j++;
+        }
+
+        if( isColFilter ){
+          fts3ColumnFilter(pFilter->iCol, &pList, &nList);
+        }
+
+        if( !isIgnoreEmpty || nList>0 ){
+          nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
+          if( nDoclist+nByte>nAlloc ){
+            char *aNew;
+            nAlloc = nDoclist+nByte*2;
+            aNew = sqlite3_realloc(aBuffer, nAlloc);
+            if( !aNew ){
+              rc = SQLITE_NOMEM;
+              goto finished;
+            }
+            aBuffer = aNew;
+          }
+          nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev);
+          iPrev = iDocid;
+          if( isRequirePos ){
+            memcpy(&aBuffer[nDoclist], pList, nList);
+            nDoclist += nList;
+            aBuffer[nDoclist++] = '\0';
+          }
+        }
+
+        fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
+      }
+
+      if( nDoclist>0 ){
+        rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist);
+        if( rc!=SQLITE_OK ) goto finished;
+      }
+    }
+
+    /* If there is a term specified to filter on, and this is not a prefix
+    ** search, return now. The callback that corresponds to the required
+    ** term (if such a term exists in the index) has already been made.
+    */
+    if( pFilter->zTerm && !isPrefix ){
+      goto finished;
+    }
+
+    for(i=0; i<nMerge; i++){
+      rc = fts3SegReaderNext(apSegment[i]);
+      if( rc!=SQLITE_OK ) goto finished;
+    }
+    fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
+  }
+
+ finished:
+  sqlite3_free(aBuffer);
+  return rc;
+}
+
+/*
+** Merge all level iLevel segments in the database into a single 
+** iLevel+1 segment. Or, if iLevel<0, merge all segments into a
+** single segment with a level equal to the numerically largest level 
+** currently present in the database.
+**
+** If this function is called with iLevel<0, but there is only one
+** segment in the database, SQLITE_DONE is returned immediately. 
+** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, 
+** an SQLite error code is returned.
+*/
+static int fts3SegmentMerge(Fts3Table *p, int iLevel){
+  int i;                          /* Iterator variable */
+  int rc;                         /* Return code */
+  int iIdx;                       /* Index of new segment */
+  int iNewLevel;                  /* Level to create new segment at */
+  sqlite3_stmt *pStmt = 0;
+  SegmentWriter *pWriter = 0;
+  int nSegment = 0;               /* Number of segments being merged */
+  Fts3SegReader **apSegment = 0;  /* Array of Segment iterators */
+  Fts3SegReader *pPending = 0;    /* Iterator for pending-terms */
+  Fts3SegFilter filter;           /* Segment term filter condition */
+
+  if( iLevel<0 ){
+    /* This call is to merge all segments in the database to a single
+    ** segment. The level of the new segment is equal to the the numerically 
+    ** greatest segment level currently present in the database. The index
+    ** of the new segment is always 0.
+    */
+    iIdx = 0;
+    rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pPending);
+    if( rc!=SQLITE_OK ) goto finished;
+    rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel);
+    if( rc!=SQLITE_OK ) goto finished;
+    nSegment += (pPending!=0);
+    if( nSegment<=1 ){
+      return SQLITE_DONE;
+    }
+  }else{
+    /* This call is to merge all segments at level iLevel. Find the next
+    ** available segment index at level iLevel+1. The call to
+    ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to 
+    ** a single iLevel+2 segment if necessary.
+    */
+    iNewLevel = iLevel+1;
+    rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
+    if( rc!=SQLITE_OK ) goto finished;
+    rc = fts3SegmentCount(p, iLevel, &nSegment);
+    if( rc!=SQLITE_OK ) goto finished;
+  }
+  assert( nSegment>0 );
+  assert( iNewLevel>=0 );
+
+  /* Allocate space for an array of pointers to segment iterators. */
+  apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment);
+  if( !apSegment ){
+    rc = SQLITE_NOMEM;
+    goto finished;
+  }
+  memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment);
+
+  /* Allocate a Fts3SegReader structure for each segment being merged. A 
+  ** Fts3SegReader stores the state data required to iterate through all 
+  ** entries on all leaves of a single segment. 
+  */
+  assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL);
+  rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0);
+  if( rc!=SQLITE_OK ) goto finished;
+  sqlite3_bind_int(pStmt, 1, iLevel);
+  for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){
+    rc = fts3SegReaderNew(p, pStmt, i, &apSegment[i]);
+    if( rc!=SQLITE_OK ){
+      goto finished;
+    }
+  }
+  rc = sqlite3_reset(pStmt);
+  if( pPending ){
+    apSegment[i] = pPending;
+    pPending = 0;
+  }
+  pStmt = 0;
+  if( rc!=SQLITE_OK ) goto finished;
+
+  memset(&filter, 0, sizeof(Fts3SegFilter));
+  filter.flags = FTS3_SEGMENT_REQUIRE_POS;
+  filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
+  rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment,
+      &filter, fts3MergeCallback, (void *)&pWriter
+  );
+  if( rc!=SQLITE_OK ) goto finished;
+
+  rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment);
+  if( rc==SQLITE_OK ){
+    rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
+  }
+
+ finished:
+  fts3SegWriterFree(pWriter);
+  if( apSegment ){
+    for(i=0; i<nSegment; i++){
+      sqlite3Fts3SegReaderFree(p, apSegment[i]);
+    }
+    sqlite3_free(apSegment);
+  }
+  sqlite3Fts3SegReaderFree(p, pPending);
+  sqlite3_reset(pStmt);
+  return rc;
+}
+
+
+/* 
+** Flush the contents of pendingTerms to a level 0 segment.
+*/
+int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
+  int rc;                         /* Return Code */
+  int idx;                        /* Index of new segment created */
+  SegmentWriter *pWriter = 0;     /* Used to write the segment */
+  Fts3SegReader *pReader = 0;     /* Used to iterate through the hash table */
+
+  /* Allocate a SegReader object to iterate through the contents of the
+  ** pending-terms table. If an error occurs, or if there are no terms
+  ** in the pending-terms table, return immediately.
+  */
+  rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pReader);
+  if( rc!=SQLITE_OK || pReader==0 ){
+    return rc;
+  }
+
+  /* Determine the next index at level 0. If level 0 is already full, this
+  ** call may merge all existing level 0 segments into a single level 1
+  ** segment.
+  */
+  rc = fts3AllocateSegdirIdx(p, 0, &idx);
+
+  /* If no errors have occured, iterate through the contents of the 
+  ** pending-terms hash table using the Fts3SegReader iterator. The callback
+  ** writes each term (along with its doclist) to the database via the
+  ** SegmentWriter handle pWriter.
+  */
+  if( rc==SQLITE_OK ){
+    void *c = (void *)&pWriter;   /* SegReaderIterate() callback context */
+    Fts3SegFilter f;              /* SegReaderIterate() parameters */
+
+    memset(&f, 0, sizeof(Fts3SegFilter));
+    f.flags = FTS3_SEGMENT_REQUIRE_POS;
+    rc = sqlite3Fts3SegReaderIterate(p, &pReader, 1, &f, fts3FlushCallback, c);
+  }
+  assert( pWriter || rc!=SQLITE_OK );
+
+  /* If no errors have occured, flush the SegmentWriter object to the
+  ** database. Then delete the SegmentWriter and Fts3SegReader objects
+  ** allocated by this function.
+  */
+  if( rc==SQLITE_OK ){
+    rc = fts3SegWriterFlush(p, pWriter, 0, idx);
+  }
+  fts3SegWriterFree(pWriter);
+  sqlite3Fts3SegReaderFree(p, pReader);
+
+  if( rc==SQLITE_OK ){
+    sqlite3Fts3PendingTermsClear(p);
+  }
+  return rc;
+}
+
+/*
+** Encode N integers as varints into a blob.
+*/
+static void fts3EncodeIntArray(
+  int N,             /* The number of integers to encode */
+  u32 *a,            /* The integer values */
+  char *zBuf,        /* Write the BLOB here */
+  int *pNBuf         /* Write number of bytes if zBuf[] used here */
+){
+  int i, j;
+  for(i=j=0; i<N; i++){
+    j += sqlite3Fts3PutVarint(&zBuf[j], (sqlite3_int64)a[i]);
+  }
+  *pNBuf = j;
+}
+
+/*
+** Decode a blob of varints into N integers
+*/
+static void fts3DecodeIntArray(
+  int N,             /* The number of integers to decode */
+  u32 *a,            /* Write the integer values */
+  const char *zBuf,  /* The BLOB containing the varints */
+  int nBuf           /* size of the BLOB */
+){
+  int i, j;
+  UNUSED_PARAMETER(nBuf);
+  for(i=j=0; i<N; i++){
+    sqlite3_int64 x;
+    j += sqlite3Fts3GetVarint(&zBuf[j], &x);
+    assert(j<=nBuf);
+    a[i] = (u32)(x & 0xffffffff);
+  }
+}
+
+/*
+** Fill in the document size auxiliary information for the matchinfo
+** structure.  The auxiliary information is:
+**
+**    N     Total number of documents in the full-text index
+**    a0    Average length of column 0 over the whole index
+**    n0    Length of column 0 on the matching row
+**    ...
+**    aM    Average length of column M over the whole index
+**    nM    Length of column M on the matching row
+**
+** The fts3MatchinfoDocsizeLocal() routine fills in the nX values.
+** The fts3MatchinfoDocsizeGlobal() routine fills in N and the aX values.
+*/
+int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor *pCur, u32 *a){
+  const char *pBlob;       /* The BLOB holding %_docsize info */
+  int nBlob;               /* Size of the BLOB */
+  sqlite3_stmt *pStmt;     /* Statement for reading and writing */
+  int i, j;                /* Loop counters */
+  sqlite3_int64 x;         /* Varint value */
+  int rc;                  /* Result code from subfunctions */
+  Fts3Table *p;            /* The FTS table */
+
+  p = (Fts3Table*)pCur->base.pVtab;
+  rc = fts3SqlStmt(p, SQL_SELECT_DOCSIZE, &pStmt, 0);
+  if( rc ){
+    return rc;
+  }
+  sqlite3_bind_int64(pStmt, 1, pCur->iPrevId);
+  if( sqlite3_step(pStmt)==SQLITE_ROW ){
+    nBlob = sqlite3_column_bytes(pStmt, 0);
+    pBlob = (const char*)sqlite3_column_blob(pStmt, 0);
+    for(i=j=0; i<p->nColumn && j<nBlob; i++){
+      j = sqlite3Fts3GetVarint(&pBlob[j], &x);
+      a[2+i*2] = (u32)(x & 0xffffffff);
+    }
+  }
+  sqlite3_reset(pStmt);
+  return SQLITE_OK; 
+}
+int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor *pCur, u32 *a){
+  const char *pBlob;       /* The BLOB holding %_stat info */
+  int nBlob;               /* Size of the BLOB */
+  sqlite3_stmt *pStmt;     /* Statement for reading and writing */
+  int i, j;                /* Loop counters */
+  sqlite3_int64 x;         /* Varint value */
+  int nDoc;                /* Number of documents */
+  int rc;                  /* Result code from subfunctions */
+  Fts3Table *p;            /* The FTS table */
+
+  p = (Fts3Table*)pCur->base.pVtab;
+  rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
+  if( rc ){
+    return rc;
+  }
+  if( sqlite3_step(pStmt)==SQLITE_ROW ){
+    nBlob = sqlite3_column_bytes(pStmt, 0);
+    pBlob = (const char*)sqlite3_column_blob(pStmt, 0);
+    j = sqlite3Fts3GetVarint(pBlob, &x);
+    a[0] = nDoc = (u32)(x & 0xffffffff);
+    for(i=0; i<p->nColumn && j<nBlob; i++){
+      j = sqlite3Fts3GetVarint(&pBlob[j], &x);
+      a[1+i*2] = ((u32)(x & 0xffffffff) + nDoc/2)/nDoc;
+    }
+  }
+  sqlite3_reset(pStmt);
+  return SQLITE_OK; 
+}
+
+/*
+** Insert the sizes (in tokens) for each column of the document
+** with docid equal to p->iPrevDocid.  The sizes are encoded as
+** a blob of varints.
+*/
+static void fts3InsertDocsize(
+  int *pRC,         /* Result code */
+  Fts3Table *p,     /* Table into which to insert */
+  u32 *aSz          /* Sizes of each column */
+){
+  char *pBlob;             /* The BLOB encoding of the document size */
+  int nBlob;               /* Number of bytes in the BLOB */
+  sqlite3_stmt *pStmt;     /* Statement used to insert the encoding */
+  int rc;                  /* Result code from subfunctions */
+
+  if( *pRC ) return;
+  pBlob = sqlite3_malloc( 10*p->nColumn );
+  if( pBlob==0 ){
+    *pRC = SQLITE_NOMEM;
+    return;
+  }
+  fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob);
+  rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0);
+  if( rc ){
+    sqlite3_free(pBlob);
+    *pRC = rc;
+    return;
+  }
+  sqlite3_bind_int64(pStmt, 1, p->iPrevDocid);
+  sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free);
+  sqlite3_step(pStmt);
+  *pRC = sqlite3_reset(pStmt);
+}
+
+/*
+** Update the 0 record of the %_stat table so that it holds a blob
+** which contains the document count followed by the cumulative
+** document sizes for all columns.
+*/
+static void fts3UpdateDocTotals(
+  int *pRC,       /* The result code */
+  Fts3Table *p,   /* Table being updated */
+  u32 *aSzIns,    /* Size increases */
+  u32 *aSzDel,    /* Size decreases */
+  int nChng       /* Change in the number of documents */
+){
+  char *pBlob;             /* Storage for BLOB written into %_stat */
+  int nBlob;               /* Size of BLOB written into %_stat */
+  u32 *a;                  /* Array of integers that becomes the BLOB */
+  sqlite3_stmt *pStmt;     /* Statement for reading and writing */
+  int i;                   /* Loop counter */
+  int rc;                  /* Result code from subfunctions */
+
+  if( *pRC ) return;
+  a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) );
+  if( a==0 ){
+    *pRC = SQLITE_NOMEM;
+    return;
+  }
+  pBlob = (char*)&a[p->nColumn+1];
+  rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
+  if( rc ){
+    sqlite3_free(a);
+    *pRC = rc;
+    return;
+  }
+  if( sqlite3_step(pStmt)==SQLITE_ROW ){
+    fts3DecodeIntArray(p->nColumn+1, a,
+         sqlite3_column_blob(pStmt, 0),
+         sqlite3_column_bytes(pStmt, 0));
+  }else{
+    memset(a, 0, sizeof(u32)*(p->nColumn+1) );
+  }
+  sqlite3_reset(pStmt);
+  if( nChng<0 && a[0]<(u32)(-nChng) ){
+    a[0] = 0;
+  }else{
+    a[0] += nChng;
+  }
+  for(i=0; i<p->nColumn; i++){
+    u32 x = a[i+1];
+    if( x+aSzIns[i] < aSzDel[i] ){
+      x = 0;
+    }else{
+      x = x + aSzIns[i] - aSzDel[i];
+    }
+    a[i+1] = x;
+  }
+  fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob);
+  rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
+  if( rc ){
+    sqlite3_free(a);
+    *pRC = rc;
+    return;
+  }
+  sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
+  sqlite3_step(pStmt);
+  *pRC = sqlite3_reset(pStmt);
+  sqlite3_free(a);
+}
+
+/*
+** Handle a 'special' INSERT of the form:
+**
+**   "INSERT INTO tbl(tbl) VALUES(<expr>)"
+**
+** Argument pVal contains the result of <expr>. Currently the only 
+** meaningful value to insert is the text 'optimize'.
+*/
+static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
+  int rc;                         /* Return Code */
+  const char *zVal = (const char *)sqlite3_value_text(pVal);
+  int nVal = sqlite3_value_bytes(pVal);
+
+  if( !zVal ){
+    return SQLITE_NOMEM;
+  }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
+    rc = fts3SegmentMerge(p, -1);
+    if( rc==SQLITE_DONE ){
+      rc = SQLITE_OK;
+    }else{
+      sqlite3Fts3PendingTermsClear(p);
+    }
+#ifdef SQLITE_TEST
+  }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
+    p->nNodeSize = atoi(&zVal[9]);
+    rc = SQLITE_OK;
+  }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
+    p->nMaxPendingData = atoi(&zVal[11]);
+    rc = SQLITE_OK;
+#endif
+  }else{
+    rc = SQLITE_ERROR;
+  }
+
+  return rc;
+}
+
+/*
+** This function does the work for the xUpdate method of FTS3 virtual
+** tables.
+*/
+int sqlite3Fts3UpdateMethod(
+  sqlite3_vtab *pVtab,            /* FTS3 vtab object */
+  int nArg,                       /* Size of argument array */
+  sqlite3_value **apVal,          /* Array of arguments */
+  sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
+){
+  Fts3Table *p = (Fts3Table *)pVtab;
+  int rc = SQLITE_OK;             /* Return Code */
+  int isRemove = 0;               /* True for an UPDATE or DELETE */
+  sqlite3_int64 iRemove = 0;      /* Rowid removed by UPDATE or DELETE */
+  u32 *aSzIns;                    /* Sizes of inserted documents */
+  u32 *aSzDel;                    /* Sizes of deleted documents */
+  int nChng = 0;                  /* Net change in number of documents */
+
+
+  /* Allocate space to hold the change in document sizes */
+  aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 );
+  if( aSzIns==0 ) return SQLITE_NOMEM;
+  aSzDel = &aSzIns[p->nColumn];
+  memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2);
+
+  /* If this is a DELETE or UPDATE operation, remove the old record. */
+  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
+    int isEmpty;
+    rc = fts3IsEmpty(p, apVal, &isEmpty);
+    if( rc==SQLITE_OK ){
+      if( isEmpty ){
+        /* Deleting this row means the whole table is empty. In this case
+        ** delete the contents of all three tables and throw away any
+        ** data in the pendingTerms hash table.
+        */
+        rc = fts3DeleteAll(p);
+      }else{
+        isRemove = 1;
+        iRemove = sqlite3_value_int64(apVal[0]);
+        rc = fts3PendingTermsDocid(p, iRemove);
+        fts3DeleteTerms(&rc, p, apVal, aSzDel);
+        fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal);
+        if( p->bHasDocsize ){
+          fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal);
+          nChng--;
+        }
+      }
+    }
+  }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){
+    sqlite3_free(aSzIns);
+    return fts3SpecialInsert(p, apVal[p->nColumn+2]);
+  }
+  
+  /* If this is an INSERT or UPDATE operation, insert the new record. */
+  if( nArg>1 && rc==SQLITE_OK ){
+    rc = fts3InsertData(p, apVal, pRowid);
+    if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){
+      rc = fts3PendingTermsDocid(p, *pRowid);
+    }
+    if( rc==SQLITE_OK ){
+      rc = fts3InsertTerms(p, apVal, aSzIns);
+    }
+    if( p->bHasDocsize ){
+      nChng++;
+      fts3InsertDocsize(&rc, p, aSzIns);
+    }
+  }
+
+  if( p->bHasDocsize ){
+    fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
+  }
+
+  sqlite3_free(aSzIns);
+  return rc;
+}
+
+/* 
+** Flush any data in the pending-terms hash table to disk. If successful,
+** merge all segments in the database (including the new segment, if 
+** there was any data to flush) into a single segment. 
+*/
+int sqlite3Fts3Optimize(Fts3Table *p){
+  int rc;
+  rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
+  if( rc==SQLITE_OK ){
+    rc = fts3SegmentMerge(p, -1);
+    if( rc==SQLITE_OK ){
+      rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
+      if( rc==SQLITE_OK ){
+        sqlite3Fts3PendingTermsClear(p);
+      }
+    }else{
+      sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
+      sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
+    }
+  }
+  return rc;
+}
+
+#endif
diff --git a/third_party/sqlite/src/ext/fts3/mkfts3amal.tcl b/third_party/sqlite/src/ext/fts3/mkfts3amal.tcl
index 0590487..0590487 100755..100644
--- a/third_party/sqlite/src/ext/fts3/mkfts3amal.tcl
+++ b/third_party/sqlite/src/ext/fts3/mkfts3amal.tcl
diff --git a/third_party/sqlite/src/ext/icu/README.txt b/third_party/sqlite/src/ext/icu/README.txt
index 5c995cc..c5cadb5 100644
--- a/third_party/sqlite/src/ext/icu/README.txt
+++ b/third_party/sqlite/src/ext/icu/README.txt
@@ -139,7 +139,7 @@ SQLite. Documentation follows.
   3.2 The SQLITE_MAX_LIKE_PATTERN_LENGTH Macro
 
     Passing very long patterns to the built-in SQLite LIKE operator can
-    cause a stack overflow. To curb this problem, SQLite defines the
+    cause excessive CPU usage. To curb this problem, SQLite defines the
     SQLITE_MAX_LIKE_PATTERN_LENGTH macro as the maximum length of a
     pattern in bytes (irrespective of encoding). The default value is
     defined in internal header file "limits.h".
@@ -167,4 +167,3 @@ SQLite. Documentation follows.
     malicious users may execute arbitrary SQL (i.e. gears), they
     should be prevented from invoking the icu_load_collation() function,
     possibly using the authorisation callback.
-
diff --git a/third_party/sqlite/src/ext/rtree/README b/third_party/sqlite/src/ext/rtree/README
index 3736f45..3736f45 100755..100644
--- a/third_party/sqlite/src/ext/rtree/README
+++ b/third_party/sqlite/src/ext/rtree/README
diff --git a/third_party/sqlite/src/ext/rtree/rtree.c b/third_party/sqlite/src/ext/rtree/rtree.c
index 5a4f570..b805676 100644
--- a/third_party/sqlite/src/ext/rtree/rtree.c
+++ b/third_party/sqlite/src/ext/rtree/rtree.c
@@ -11,8 +11,45 @@
 *************************************************************************
 ** This file contains code for implementations of the r-tree and r*-tree
 ** algorithms packaged as an SQLite virtual table module.
+*/
+
+/*
+** Database Format of R-Tree Tables
+** --------------------------------
+**
+** The data structure for a single virtual r-tree table is stored in three 
+** native SQLite tables declared as follows. In each case, the '%' character
+** in the table name is replaced with the user-supplied name of the r-tree
+** table.
+**
+**   CREATE TABLE %_node(nodeno INTEGER PRIMARY KEY, data BLOB)
+**   CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER)
+**   CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER)
+**
+** The data for each node of the r-tree structure is stored in the %_node
+** table. For each node that is not the root node of the r-tree, there is
+** an entry in the %_parent table associating the node with its parent.
+** And for each row of data in the table, there is an entry in the %_rowid
+** table that maps from the entries rowid to the id of the node that it
+** is stored on.
+**
+** The root node of an r-tree always exists, even if the r-tree table is
+** empty. The nodeno of the root node is always 1. All other nodes in the
+** table must be the same size as the root node. The content of each node
+** is formatted as follows:
+**
+**   1. If the node is the root node (node 1), then the first 2 bytes
+**      of the node contain the tree depth as a big-endian integer.
+**      For non-root nodes, the first 2 bytes are left unused.
+**
+**   2. The next 2 bytes contain the number of entries currently 
+**      stored in the node.
 **
-** $Id: rtree.c,v 1.14 2009/08/06 18:36:47 danielk1977 Exp $
+**   3. The remainder of the node contains the node entries. Each entry
+**      consists of a single 8-byte integer followed by an even number
+**      of 4-byte coordinates. For leaf nodes the integer is the rowid
+**      of a record. For internal nodes it is the node number of a
+**      child page.
 */
 
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE)
@@ -55,6 +92,9 @@
   #define AssignCells splitNodeStartree
 #endif
 
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
+# define NDEBUG 1
+#endif
 
 #ifndef SQLITE_CORE
   #include "sqlite3ext.h"
@@ -67,6 +107,7 @@
 #include <assert.h>
 
 #ifndef SQLITE_AMALGAMATION
+#include "sqlite3rtree.h"
 typedef sqlite3_int64 i64;
 typedef unsigned char u8;
 typedef unsigned int u32;
@@ -77,6 +118,8 @@ typedef struct RtreeCursor RtreeCursor;
 typedef struct RtreeNode RtreeNode;
 typedef struct RtreeCell RtreeCell;
 typedef struct RtreeConstraint RtreeConstraint;
+typedef struct RtreeMatchArg RtreeMatchArg;
+typedef struct RtreeGeomCallback RtreeGeomCallback;
 typedef union RtreeCoord RtreeCoord;
 
 /* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */
@@ -146,6 +189,15 @@ struct Rtree {
 #define RTREE_REINSERT(p) RTREE_MINCELLS(p)
 #define RTREE_MAXCELLS 51
 
+/*
+** The smallest possible node-size is (512-64)==448 bytes. And the largest
+** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates).
+** Therefore all non-root nodes must contain at least 3 entries. Since 
+** 2^40 is greater than 2^64, an r-tree structure always has a depth of
+** 40 or less.
+*/
+#define RTREE_MAX_DEPTH 40
+
 /* 
 ** An rtree cursor object.
 */
@@ -178,35 +230,23 @@ union RtreeCoord {
 ** A search constraint.
 */
 struct RtreeConstraint {
-  int iCoord;                       /* Index of constrained coordinate */
-  int op;                           /* Constraining operation */
-  double rValue;                    /* Constraint value. */
+  int iCoord;                     /* Index of constrained coordinate */
+  int op;                         /* Constraining operation */
+  double rValue;                  /* Constraint value. */
+  int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *);
+  sqlite3_rtree_geometry *pGeom;  /* Constraint callback argument for a MATCH */
 };
 
 /* Possible values for RtreeConstraint.op */
-#define RTREE_EQ 0x41
-#define RTREE_LE 0x42
-#define RTREE_LT 0x43
-#define RTREE_GE 0x44
-#define RTREE_GT 0x45
+#define RTREE_EQ    0x41
+#define RTREE_LE    0x42
+#define RTREE_LT    0x43
+#define RTREE_GE    0x44
+#define RTREE_GT    0x45
+#define RTREE_MATCH 0x46
 
 /* 
 ** An rtree structure node.
-**
-** Data format (RtreeNode.zData):
-**
-**   1. If the node is the root node (node 1), then the first 2 bytes
-**      of the node contain the tree depth as a big-endian integer.
-**      For non-root nodes, the first 2 bytes are left unused.
-**
-**   2. The next 2 bytes contain the number of entries currently 
-**      stored in the node.
-**
-**   3. The remainder of the node contains the node entries. Each entry
-**      consists of a single 8-byte integer followed by an even number
-**      of 4-byte coordinates. For leaf nodes the integer is the rowid
-**      of a record. For internal nodes it is the node number of a
-**      child page.
 */
 struct RtreeNode {
   RtreeNode *pParent;               /* Parent node */
@@ -226,6 +266,40 @@ struct RtreeCell {
   RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2];
 };
 
+
+/*
+** Value for the first field of every RtreeMatchArg object. The MATCH
+** operator tests that the first field of a blob operand matches this
+** value to avoid operating on invalid blobs (which could cause a segfault).
+*/
+#define RTREE_GEOMETRY_MAGIC 0x891245AB
+
+/*
+** An instance of this structure must be supplied as a blob argument to
+** the right-hand-side of an SQL MATCH operator used to constrain an
+** r-tree query.
+*/
+struct RtreeMatchArg {
+  u32 magic;                      /* Always RTREE_GEOMETRY_MAGIC */
+  int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *);
+  void *pContext;
+  int nParam;
+  double aParam[1];
+};
+
+/*
+** When a geometry callback is created (see sqlite3_rtree_geometry_callback),
+** a single instance of the following structure is allocated. It is used
+** as the context for the user-function created by by s_r_g_c(). The object
+** is eventually deleted by the destructor mechanism provided by
+** sqlite3_create_function_v2() (which is called by s_r_g_c() to create
+** the geometry callback function).
+*/
+struct RtreeGeomCallback {
+  int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *);
+  void *pContext;
+};
+
 #ifndef MAX
 # define MAX(x,y) ((x) < (y) ? (y) : (x))
 #endif
@@ -308,10 +382,8 @@ static void nodeReference(RtreeNode *p){
 ** Clear the content of node p (set all bytes to 0x00).
 */
 static void nodeZero(Rtree *pRtree, RtreeNode *p){
-  if( p ){
-    memset(&p->zData[2], 0, pRtree->iNodeSize-2);
-    p->isDirty = 1;
-  }
+  memset(&p->zData[2], 0, pRtree->iNodeSize-2);
+  p->isDirty = 1;
 }
 
 /*
@@ -331,7 +403,6 @@ static int nodeHash(i64 iNode){
 */
 static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){
   RtreeNode *p;
-  assert( iNode!=0 );
   for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext);
   return p;
 }
@@ -340,13 +411,11 @@ static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){
 ** Add node pNode to the node hash table.
 */
 static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){
-  if( pNode ){
-    int iHash;
-    assert( pNode->pNext==0 );
-    iHash = nodeHash(pNode->iNode);
-    pNode->pNext = pRtree->aHash[iHash];
-    pRtree->aHash[iHash] = pNode;
-  }
+  int iHash;
+  assert( pNode->pNext==0 );
+  iHash = nodeHash(pNode->iNode);
+  pNode->pNext = pRtree->aHash[iHash];
+  pRtree->aHash[iHash] = pNode;
 }
 
 /*
@@ -368,11 +437,11 @@ static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){
 ** assigned a node number when nodeWrite() is called to write the
 ** node contents out to the database.
 */
-static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent, int zero){
+static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){
   RtreeNode *pNode;
   pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize);
   if( pNode ){
-    memset(pNode, 0, sizeof(RtreeNode) + (zero?pRtree->iNodeSize:0));
+    memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize);
     pNode->zData = (u8 *)&pNode[1];
     pNode->nRef = 1;
     pNode->pParent = pParent;
@@ -393,6 +462,7 @@ nodeAcquire(
   RtreeNode **ppNode         /* OUT: Acquired node */
 ){
   int rc;
+  int rc2 = SQLITE_OK;
   RtreeNode *pNode;
 
   /* Check if the requested node is already in the hash table. If so,
@@ -409,38 +479,63 @@ nodeAcquire(
     return SQLITE_OK;
   }
 
-  pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize);
-  if( !pNode ){
-    *ppNode = 0;
-    return SQLITE_NOMEM;
-  }
-  pNode->pParent = pParent;
-  pNode->zData = (u8 *)&pNode[1];
-  pNode->nRef = 1;
-  pNode->iNode = iNode;
-  pNode->isDirty = 0;
-  pNode->pNext = 0;
-
   sqlite3_bind_int64(pRtree->pReadNode, 1, iNode);
   rc = sqlite3_step(pRtree->pReadNode);
   if( rc==SQLITE_ROW ){
     const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0);
-    memcpy(pNode->zData, zBlob, pRtree->iNodeSize);
-    nodeReference(pParent);
-  }else{
-    sqlite3_free(pNode);
-    pNode = 0;
+    if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){
+      pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize);
+      if( !pNode ){
+        rc2 = SQLITE_NOMEM;
+      }else{
+        pNode->pParent = pParent;
+        pNode->zData = (u8 *)&pNode[1];
+        pNode->nRef = 1;
+        pNode->iNode = iNode;
+        pNode->isDirty = 0;
+        pNode->pNext = 0;
+        memcpy(pNode->zData, zBlob, pRtree->iNodeSize);
+        nodeReference(pParent);
+      }
+    }
   }
-
-  *ppNode = pNode;
   rc = sqlite3_reset(pRtree->pReadNode);
+  if( rc==SQLITE_OK ) rc = rc2;
 
-  if( rc==SQLITE_OK && iNode==1 ){
+  /* If the root node was just loaded, set pRtree->iDepth to the height
+  ** of the r-tree structure. A height of zero means all data is stored on
+  ** the root node. A height of one means the children of the root node
+  ** are the leaves, and so on. If the depth as specified on the root node
+  ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt.
+  */
+  if( pNode && iNode==1 ){
     pRtree->iDepth = readInt16(pNode->zData);
+    if( pRtree->iDepth>RTREE_MAX_DEPTH ){
+      rc = SQLITE_CORRUPT;
+    }
   }
 
-  assert( (rc==SQLITE_OK && pNode) || (pNode==0 && rc!=SQLITE_OK) );
-  nodeHashInsert(pRtree, pNode);
+  /* If no error has occurred so far, check if the "number of entries"
+  ** field on the node is too large. If so, set the return code to 
+  ** SQLITE_CORRUPT.
+  */
+  if( pNode && rc==SQLITE_OK ){
+    if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){
+      rc = SQLITE_CORRUPT;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    if( pNode!=0 ){
+      nodeHashInsert(pRtree, pNode);
+    }else{
+      rc = SQLITE_CORRUPT;
+    }
+    *ppNode = pNode;
+  }else{
+    sqlite3_free(pNode);
+    *ppNode = 0;
+  }
 
   return rc;
 }
@@ -493,8 +588,7 @@ nodeInsertCell(
   nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell;
   nCell = NCELL(pNode);
 
-  assert(nCell<=nMaxCell);
-
+  assert( nCell<=nMaxCell );
   if( nCell<nMaxCell ){
     nodeOverwriteCell(pRtree, pNode, pCell, nCell);
     writeInt16(&pNode->zData[2], nCell+1);
@@ -714,6 +808,25 @@ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
   return rc;
 }
 
+
+/*
+** Free the RtreeCursor.aConstraint[] array and its contents.
+*/
+static void freeCursorConstraints(RtreeCursor *pCsr){
+  if( pCsr->aConstraint ){
+    int i;                        /* Used to iterate through constraint array */
+    for(i=0; i<pCsr->nConstraint; i++){
+      sqlite3_rtree_geometry *pGeom = pCsr->aConstraint[i].pGeom;
+      if( pGeom ){
+        if( pGeom->xDelUser ) pGeom->xDelUser(pGeom->pUser);
+        sqlite3_free(pGeom);
+      }
+    }
+    sqlite3_free(pCsr->aConstraint);
+    pCsr->aConstraint = 0;
+  }
+}
+
 /* 
 ** Rtree virtual table module xClose method.
 */
@@ -721,7 +834,7 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){
   Rtree *pRtree = (Rtree *)(cur->pVtab);
   int rc;
   RtreeCursor *pCsr = (RtreeCursor *)cur;
-  sqlite3_free(pCsr->aConstraint);
+  freeCursorConstraints(pCsr);
   rc = nodeRelease(pRtree, pCsr->pNode);
   sqlite3_free(pCsr);
   return rc;
@@ -738,13 +851,39 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){
   return (pCsr->pNode==0);
 }
 
+/*
+** The r-tree constraint passed as the second argument to this function is
+** guaranteed to be a MATCH constraint.
+*/
+static int testRtreeGeom(
+  Rtree *pRtree,                  /* R-Tree object */
+  RtreeConstraint *pConstraint,   /* MATCH constraint to test */
+  RtreeCell *pCell,               /* Cell to test */
+  int *pbRes                      /* OUT: Test result */
+){
+  int i;
+  double aCoord[RTREE_MAX_DIMENSIONS*2];
+  int nCoord = pRtree->nDim*2;
+
+  assert( pConstraint->op==RTREE_MATCH );
+  assert( pConstraint->pGeom );
+
+  for(i=0; i<nCoord; i++){
+    aCoord[i] = DCOORD(pCell->aCoord[i]);
+  }
+  return pConstraint->xGeom(pConstraint->pGeom, nCoord, aCoord, pbRes);
+}
+
 /* 
 ** Cursor pCursor currently points to a cell in a non-leaf page.
-** Return true if the sub-tree headed by the cell is filtered
+** Set *pbEof to true if the sub-tree headed by the cell is filtered
 ** (excluded) by the constraints in the pCursor->aConstraint[] 
 ** array, or false otherwise.
+**
+** Return SQLITE_OK if successful or an SQLite error code if an error
+** occurs within a geometry callback.
 */
-static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor){
+static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){
   RtreeCell cell;
   int ii;
   int bRes = 0;
@@ -756,31 +895,55 @@ static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor){
     double cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]);
 
     assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE 
-        || p->op==RTREE_GT || p->op==RTREE_EQ
+        || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH
     );
 
     switch( p->op ){
-      case RTREE_LE: case RTREE_LT: bRes = p->rValue<cell_min; break;
-      case RTREE_GE: case RTREE_GT: bRes = p->rValue>cell_max; break;
-      case RTREE_EQ: 
+      case RTREE_LE: case RTREE_LT: 
+        bRes = p->rValue<cell_min; 
+        break;
+
+      case RTREE_GE: case RTREE_GT: 
+        bRes = p->rValue>cell_max; 
+        break;
+
+      case RTREE_EQ:
         bRes = (p->rValue>cell_max || p->rValue<cell_min);
         break;
+
+      default: {
+        int rc;
+        assert( p->op==RTREE_MATCH );
+        rc = testRtreeGeom(pRtree, p, &cell, &bRes);
+        if( rc!=SQLITE_OK ){
+          return rc;
+        }
+        bRes = !bRes;
+        break;
+      }
     }
   }
 
-  return bRes;
+  *pbEof = bRes;
+  return SQLITE_OK;
 }
 
 /* 
-** Return true if the cell that cursor pCursor currently points to
+** Test if the cell that cursor pCursor currently points to
 ** would be filtered (excluded) by the constraints in the 
-** pCursor->aConstraint[] array, or false otherwise.
+** pCursor->aConstraint[] array. If so, set *pbEof to true before
+** returning. If the cell is not filtered (excluded) by the constraints,
+** set pbEof to zero.
+**
+** Return SQLITE_OK if successful or an SQLite error code if an error
+** occurs within a geometry callback.
 **
 ** This function assumes that the cell is part of a leaf node.
 */
-static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor){
+static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){
   RtreeCell cell;
   int ii;
+  *pbEof = 0;
 
   nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell);
   for(ii=0; ii<pCursor->nConstraint; ii++){
@@ -788,7 +951,7 @@ static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor){
     double coord = DCOORD(cell.aCoord[p->iCoord]);
     int res;
     assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE 
-        || p->op==RTREE_GT || p->op==RTREE_EQ
+        || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH
     );
     switch( p->op ){
       case RTREE_LE: res = (coord<=p->rValue); break;
@@ -796,12 +959,24 @@ static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor){
       case RTREE_GE: res = (coord>=p->rValue); break;
       case RTREE_GT: res = (coord>p->rValue);  break;
       case RTREE_EQ: res = (coord==p->rValue); break;
+      default: {
+        int rc;
+        assert( p->op==RTREE_MATCH );
+        rc = testRtreeGeom(pRtree, p, &cell, &res);
+        if( rc!=SQLITE_OK ){
+          return rc;
+        }
+        break;
+      }
     }
 
-    if( !res ) return 1;
+    if( !res ){
+      *pbEof = 1;
+      return SQLITE_OK;
+    }
   }
 
-  return 0;
+  return SQLITE_OK;
 }
 
 /*
@@ -828,13 +1003,13 @@ static int descendToCell(
   assert( iHeight>=0 );
 
   if( iHeight==0 ){
-    isEof = testRtreeEntry(pRtree, pCursor);
+    rc = testRtreeEntry(pRtree, pCursor, &isEof);
   }else{
-    isEof = testRtreeCell(pRtree, pCursor);
+    rc = testRtreeCell(pRtree, pCursor, &isEof);
   }
-  if( isEof || iHeight==0 ){
+  if( rc!=SQLITE_OK || isEof || iHeight==0 ){
     *pEof = isEof;
-    return SQLITE_OK;
+    return rc;
   }
 
   iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell);
@@ -870,24 +1045,34 @@ static int descendToCell(
 ** One of the cells in node pNode is guaranteed to have a 64-bit 
 ** integer value equal to iRowid. Return the index of this cell.
 */
-static int nodeRowidIndex(Rtree *pRtree, RtreeNode *pNode, i64 iRowid){
+static int nodeRowidIndex(
+  Rtree *pRtree, 
+  RtreeNode *pNode, 
+  i64 iRowid,
+  int *piIndex
+){
   int ii;
-  for(ii=0; nodeGetRowid(pRtree, pNode, ii)!=iRowid; ii++){
-    assert( ii<(NCELL(pNode)-1) );
+  int nCell = NCELL(pNode);
+  for(ii=0; ii<nCell; ii++){
+    if( nodeGetRowid(pRtree, pNode, ii)==iRowid ){
+      *piIndex = ii;
+      return SQLITE_OK;
+    }
   }
-  return ii;
+  return SQLITE_CORRUPT;
 }
 
 /*
 ** Return the index of the cell containing a pointer to node pNode
 ** in its parent. If pNode is the root node, return -1.
 */
-static int nodeParentIndex(Rtree *pRtree, RtreeNode *pNode){
+static int nodeParentIndex(Rtree *pRtree, RtreeNode *pNode, int *piIndex){
   RtreeNode *pParent = pNode->pParent;
   if( pParent ){
-    return nodeRowidIndex(pRtree, pParent, pNode->iNode);
+    return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex);
   }
-  return -1;
+  *piIndex = -1;
+  return SQLITE_OK;
 }
 
 /* 
@@ -898,13 +1083,17 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){
   RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor;
   int rc = SQLITE_OK;
 
+  /* RtreeCursor.pNode must not be NULL. If is is NULL, then this cursor is
+  ** already at EOF. It is against the rules to call the xNext() method of
+  ** a cursor that has already reached EOF.
+  */
+  assert( pCsr->pNode );
+
   if( pCsr->iStrategy==1 ){
     /* This "scan" is a direct lookup by rowid. There is no next entry. */
     nodeRelease(pRtree, pCsr->pNode);
     pCsr->pNode = 0;
-  }
-
-  else if( pCsr->pNode ){
+  }else{
     /* Move to the next entry that matches the configured constraints. */
     int iHeight = 0;
     while( pCsr->pNode ){
@@ -918,7 +1107,10 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){
         }
       }
       pCsr->pNode = pNode->pParent;
-      pCsr->iCell = nodeParentIndex(pRtree, pNode);
+      rc = nodeParentIndex(pRtree, pNode, &pCsr->iCell);
+      if( rc!=SQLITE_OK ){
+        return rc;
+      }
       nodeReference(pCsr->pNode);
       nodeRelease(pRtree, pNode);
       iHeight++;
@@ -986,6 +1178,51 @@ static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){
   return rc;
 }
 
+/*
+** This function is called to configure the RtreeConstraint object passed
+** as the second argument for a MATCH constraint. The value passed as the
+** first argument to this function is the right-hand operand to the MATCH
+** operator.
+*/
+static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){
+  RtreeMatchArg *p;
+  sqlite3_rtree_geometry *pGeom;
+  int nBlob;
+
+  /* Check that value is actually a blob. */
+  if( !sqlite3_value_type(pValue)==SQLITE_BLOB ) return SQLITE_ERROR;
+
+  /* Check that the blob is roughly the right size. */
+  nBlob = sqlite3_value_bytes(pValue);
+  if( nBlob<sizeof(RtreeMatchArg) 
+   || ((nBlob-sizeof(RtreeMatchArg))%sizeof(double))!=0
+  ){
+    return SQLITE_ERROR;
+  }
+
+  pGeom = (sqlite3_rtree_geometry *)sqlite3_malloc(
+      sizeof(sqlite3_rtree_geometry) + nBlob
+  );
+  if( !pGeom ) return SQLITE_NOMEM;
+  memset(pGeom, 0, sizeof(sqlite3_rtree_geometry));
+  p = (RtreeMatchArg *)&pGeom[1];
+
+  memcpy(p, sqlite3_value_blob(pValue), nBlob);
+  if( p->magic!=RTREE_GEOMETRY_MAGIC 
+   || nBlob!=(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double))
+  ){
+    sqlite3_free(pGeom);
+    return SQLITE_ERROR;
+  }
+
+  pGeom->pContext = p->pContext;
+  pGeom->nParam = p->nParam;
+  pGeom->aParam = p->aParam;
+
+  pCons->xGeom = p->xGeom;
+  pCons->pGeom = pGeom;
+  return SQLITE_OK;
+}
 
 /* 
 ** Rtree virtual table module xFilter method.
@@ -1004,8 +1241,7 @@ static int rtreeFilter(
 
   rtreeReference(pRtree);
 
-  sqlite3_free(pCsr->aConstraint);
-  pCsr->aConstraint = 0;
+  freeCursorConstraints(pCsr);
   pCsr->iStrategy = idxNum;
 
   if( idxNum==1 ){
@@ -1014,8 +1250,9 @@ static int rtreeFilter(
     i64 iRowid = sqlite3_value_int64(argv[0]);
     rc = findLeafNode(pRtree, iRowid, &pLeaf);
     pCsr->pNode = pLeaf; 
-    if( pLeaf && rc==SQLITE_OK ){
-      pCsr->iCell = nodeRowidIndex(pRtree, pLeaf, iRowid);
+    if( pLeaf ){
+      assert( rc==SQLITE_OK );
+      rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &pCsr->iCell);
     }
   }else{
     /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array 
@@ -1027,12 +1264,24 @@ static int rtreeFilter(
       if( !pCsr->aConstraint ){
         rc = SQLITE_NOMEM;
       }else{
+        memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc);
         assert( (idxStr==0 && argc==0) || strlen(idxStr)==argc*2 );
         for(ii=0; ii<argc; ii++){
           RtreeConstraint *p = &pCsr->aConstraint[ii];
           p->op = idxStr[ii*2];
           p->iCoord = idxStr[ii*2+1]-'a';
-          p->rValue = sqlite3_value_double(argv[ii]);
+          if( p->op==RTREE_MATCH ){
+            /* A MATCH operator. The right-hand-side must be a blob that
+            ** can be cast into an RtreeMatchArg object. One created using
+            ** an sqlite3_rtree_geometry_callback() SQL user function.
+            */
+            rc = deserializeGeometry(argv[ii], p);
+            if( rc!=SQLITE_OK ){
+              break;
+            }
+          }else{
+            p->rValue = sqlite3_value_double(argv[ii]);
+          }
         }
       }
     }
@@ -1073,11 +1322,10 @@ static int rtreeFilter(
 **   idxNum     idxStr        Strategy
 **   ------------------------------------------------
 **     1        Unused        Direct lookup by rowid.
-**     2        See below     R-tree query.
-**     3        Unused        Full table scan.
+**     2        See below     R-tree query or full-table scan.
 **   ------------------------------------------------
 **
-** If strategy 1 or 3 is used, then idxStr is not meaningful. If strategy
+** If strategy 1 is used, then idxStr is not meaningful. If strategy
 ** 2 is used, idxStr is formatted to contain 2 bytes for each 
 ** constraint used. The first two bytes of idxStr correspond to 
 ** the constraint in sqlite3_index_info.aConstraintUsage[] with
@@ -1093,6 +1341,7 @@ static int rtreeFilter(
 **      <        0x43 ('C')
 **     >=        0x44 ('D')
 **      >        0x45 ('E')
+**   MATCH       0x46 ('F')
 **   ----------------------
 **
 ** The second of each pair of bytes identifies the coordinate column
@@ -1131,7 +1380,9 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
       return SQLITE_OK;
     }
 
-    if( p->usable && p->iColumn>0 ){
+    if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){
+      int j, opmsk;
+      static const unsigned char compatible[] = { 0, 0, 1, 1, 2, 2 };
       u8 op = 0;
       switch( p->op ){
         case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break;
@@ -1139,31 +1390,33 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
         case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break;
         case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break;
         case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break;
+        default:
+          assert( p->op==SQLITE_INDEX_CONSTRAINT_MATCH );
+          op = RTREE_MATCH; 
+          break;
       }
-      if( op ){
-        /* Make sure this particular constraint has not been used before.
-        ** If it has been used before, ignore it.
-        **
-        ** A <= or < can be used if there is a prior >= or >.
-        ** A >= or > can be used if there is a prior < or <=.
-        ** A <= or < is disqualified if there is a prior <=, <, or ==.
-        ** A >= or > is disqualified if there is a prior >=, >, or ==.
-        ** A == is disqualifed if there is any prior constraint.
-        */
-        int j, opmsk;
-        static const unsigned char compatible[] = { 0, 0, 1, 1, 2, 2 };
-        assert( compatible[RTREE_EQ & 7]==0 );
-        assert( compatible[RTREE_LT & 7]==1 );
-        assert( compatible[RTREE_LE & 7]==1 );
-        assert( compatible[RTREE_GT & 7]==2 );
-        assert( compatible[RTREE_GE & 7]==2 );
-        cCol = p->iColumn - 1 + 'a';
-        opmsk = compatible[op & 7];
-        for(j=0; j<iIdx; j+=2){
-          if( zIdxStr[j+1]==cCol && (compatible[zIdxStr[j] & 7] & opmsk)!=0 ){
-            op = 0;
-            break;
-          }
+      assert( op!=0 );
+
+      /* Make sure this particular constraint has not been used before.
+      ** If it has been used before, ignore it.
+      **
+      ** A <= or < can be used if there is a prior >= or >.
+      ** A >= or > can be used if there is a prior < or <=.
+      ** A <= or < is disqualified if there is a prior <=, <, or ==.
+      ** A >= or > is disqualified if there is a prior >=, >, or ==.
+      ** A == is disqualifed if there is any prior constraint.
+      */
+      assert( compatible[RTREE_EQ & 7]==0 );
+      assert( compatible[RTREE_LT & 7]==1 );
+      assert( compatible[RTREE_LE & 7]==1 );
+      assert( compatible[RTREE_GT & 7]==2 );
+      assert( compatible[RTREE_GE & 7]==2 );
+      cCol = p->iColumn - 1 + 'a';
+      opmsk = compatible[op & 7];
+      for(j=0; j<iIdx; j+=2){
+        if( zIdxStr[j+1]==cCol && (compatible[zIdxStr[j] & 7] & opmsk)!=0 ){
+          op = 0;
+          break;
         }
       }
       if( op ){
@@ -1271,7 +1524,12 @@ static float cellOverlap(
   int ii;
   float overlap = 0.0;
   for(ii=0; ii<nCell; ii++){
-    if( ii!=iExclude ){
+#if VARIANT_RSTARTREE_CHOOSESUBTREE
+    if( ii!=iExclude )
+#else
+    assert( iExclude==-1 );
+#endif
+    {
       int jj;
       float o = 1.0;
       for(jj=0; jj<(pRtree->nDim*2); jj+=2){
@@ -1364,22 +1622,31 @@ static int ChooseLeaf(
     ** the smallest area.
     */
     for(iCell=0; iCell<nCell; iCell++){
+      int bBest = 0;
       float growth;
       float area;
       float overlap = 0.0;
       nodeGetCell(pRtree, pNode, iCell, &cell);
       growth = cellGrowth(pRtree, &cell, pCell);
       area = cellArea(pRtree, &cell);
+
 #if VARIANT_RSTARTREE_CHOOSESUBTREE
       if( ii==(pRtree->iDepth-1) ){
         overlap = cellOverlapEnlargement(pRtree,&cell,pCell,aCell,nCell,iCell);
       }
-#endif
       if( (iCell==0) 
        || (overlap<fMinOverlap) 
        || (overlap==fMinOverlap && growth<fMinGrowth)
        || (overlap==fMinOverlap && growth==fMinGrowth && area<fMinArea)
       ){
+        bBest = 1;
+      }
+#else
+      if( iCell==0||growth<fMinGrowth||(growth==fMinGrowth && area<fMinArea) ){
+        bBest = 1;
+      }
+#endif
+      if( bBest ){
         fMinOverlap = overlap;
         fMinGrowth = growth;
         fMinArea = area;
@@ -1402,16 +1669,20 @@ static int ChooseLeaf(
 ** the node pNode. This function updates the bounding box cells in
 ** all ancestor elements.
 */
-static void AdjustTree(
+static int AdjustTree(
   Rtree *pRtree,                    /* Rtree table */
   RtreeNode *pNode,                 /* Adjust ancestry of this node. */
   RtreeCell *pCell                  /* This cell was just inserted */
 ){
   RtreeNode *p = pNode;
   while( p->pParent ){
-    RtreeCell cell;
     RtreeNode *pParent = p->pParent;
-    int iCell = nodeParentIndex(pRtree, p);
+    RtreeCell cell;
+    int iCell;
+
+    if( nodeParentIndex(pRtree, p, &iCell) ){
+      return SQLITE_CORRUPT;
+    }
 
     nodeGetCell(pRtree, pParent, iCell, &cell);
     if( !cellContains(pRtree, &cell, pCell) ){
@@ -1421,6 +1692,7 @@ static void AdjustTree(
  
     p = pParent;
   }
+  return SQLITE_OK;
 }
 
 /*
@@ -1486,8 +1758,8 @@ static void LinearPickSeeds(
   ** variables iLeftSeek and iRightSeed.
   */
   for(i=0; i<pRtree->nDim; i++){
-    float x1 = aCell[0].aCoord[i*2];
-    float x2 = aCell[0].aCoord[i*2+1];
+    float x1 = DCOORD(aCell[0].aCoord[i*2]);
+    float x2 = DCOORD(aCell[0].aCoord[i*2+1]);
     float x3 = x1;
     float x4 = x2;
     int jj;
@@ -1496,8 +1768,8 @@ static void LinearPickSeeds(
     int iCellRight = 0;
 
     for(jj=1; jj<nCell; jj++){
-      float left = aCell[jj].aCoord[i*2];
-      float right = aCell[jj].aCoord[i*2+1];
+      float left = DCOORD(aCell[jj].aCoord[i*2]);
+      float right = DCOORD(aCell[jj].aCoord[i*2+1]);
 
       if( left<x1 ) x1 = left;
       if( right>x4 ) x4 = right;
@@ -1855,6 +2127,9 @@ static int splitNodeGuttman(
   int i;
 
   aiUsed = sqlite3_malloc(sizeof(int)*nCell);
+  if( !aiUsed ){
+    return SQLITE_NOMEM;
+  }
   memset(aiUsed, 0, sizeof(int)*nCell);
 
   PickSeeds(pRtree, aCell, nCell, &iLeftSeed, &iRightSeed);
@@ -1946,14 +2221,14 @@ static int SplitNode(
   nCell++;
 
   if( pNode->iNode==1 ){
-    pRight = nodeNew(pRtree, pNode, 1);
-    pLeft = nodeNew(pRtree, pNode, 1);
+    pRight = nodeNew(pRtree, pNode);
+    pLeft = nodeNew(pRtree, pNode);
     pRtree->iDepth++;
     pNode->isDirty = 1;
     writeInt16(pNode->zData, pRtree->iDepth);
   }else{
     pLeft = pNode;
-    pRight = nodeNew(pRtree, pLeft->pParent, 1);
+    pRight = nodeNew(pRtree, pLeft->pParent);
     nodeReference(pLeft);
   }
 
@@ -1970,8 +2245,12 @@ static int SplitNode(
     goto splitnode_out;
   }
 
-  /* Ensure both child nodes have node numbers assigned to them. */
-  if( (0==pRight->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)))
+  /* Ensure both child nodes have node numbers assigned to them by calling
+  ** nodeWrite(). Node pRight always needs a node number, as it was created
+  ** by nodeNew() above. But node pLeft sometimes already has a node number.
+  ** In this case avoid the all to nodeWrite().
+  */
+  if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight))
    || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft)))
   ){
     goto splitnode_out;
@@ -1987,9 +2266,15 @@ static int SplitNode(
     }
   }else{
     RtreeNode *pParent = pLeft->pParent;
-    int iCell = nodeParentIndex(pRtree, pLeft);
-    nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell);
-    AdjustTree(pRtree, pParent, &leftbbox);
+    int iCell;
+    rc = nodeParentIndex(pRtree, pLeft, &iCell);
+    if( rc==SQLITE_OK ){
+      nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell);
+      rc = AdjustTree(pRtree, pParent, &leftbbox);
+    }
+    if( rc!=SQLITE_OK ){
+      goto splitnode_out;
+    }
   }
   if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){
     goto splitnode_out;
@@ -2033,20 +2318,43 @@ splitnode_out:
   return rc;
 }
 
+/*
+** If node pLeaf is not the root of the r-tree and its pParent pointer is 
+** still NULL, load all ancestor nodes of pLeaf into memory and populate
+** the pLeaf->pParent chain all the way up to the root node.
+**
+** This operation is required when a row is deleted (or updated - an update
+** is implemented as a delete followed by an insert). SQLite provides the
+** rowid of the row to delete, which can be used to find the leaf on which
+** the entry resides (argument pLeaf). Once the leaf is located, this 
+** function is called to determine its ancestry.
+*/
 static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){
   int rc = SQLITE_OK;
-  if( pLeaf->iNode!=1 && pLeaf->pParent==0 ){
-    sqlite3_bind_int64(pRtree->pReadParent, 1, pLeaf->iNode);
-    if( sqlite3_step(pRtree->pReadParent)==SQLITE_ROW ){
-      i64 iNode = sqlite3_column_int64(pRtree->pReadParent, 0);
-      rc = nodeAcquire(pRtree, iNode, 0, &pLeaf->pParent);
-    }else{
-      rc = SQLITE_ERROR;
-    }
-    sqlite3_reset(pRtree->pReadParent);
-    if( rc==SQLITE_OK ){
-      rc = fixLeafParent(pRtree, pLeaf->pParent);
+  RtreeNode *pChild = pLeaf;
+  while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){
+    int rc2 = SQLITE_OK;          /* sqlite3_reset() return code */
+    sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode);
+    rc = sqlite3_step(pRtree->pReadParent);
+    if( rc==SQLITE_ROW ){
+      RtreeNode *pTest;           /* Used to test for reference loops */
+      i64 iNode;                  /* Node number of parent node */
+
+      /* Before setting pChild->pParent, test that we are not creating a
+      ** loop of references (as we would if, say, pChild==pParent). We don't
+      ** want to do this as it leads to a memory leak when trying to delete
+      ** the referenced counted node structures.
+      */
+      iNode = sqlite3_column_int64(pRtree->pReadParent, 0);
+      for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent);
+      if( !pTest ){
+        rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent);
+      }
     }
+    rc = sqlite3_reset(pRtree->pReadParent);
+    if( rc==SQLITE_OK ) rc = rc2;
+    if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT;
+    pChild = pChild->pParent;
   }
   return rc;
 }
@@ -2055,18 +2363,24 @@ static int deleteCell(Rtree *, RtreeNode *, int, int);
 
 static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){
   int rc;
+  int rc2;
   RtreeNode *pParent;
   int iCell;
 
   assert( pNode->nRef==1 );
 
   /* Remove the entry in the parent cell. */
-  iCell = nodeParentIndex(pRtree, pNode);
-  pParent = pNode->pParent;
-  pNode->pParent = 0;
-  if( SQLITE_OK!=(rc = deleteCell(pRtree, pParent, iCell, iHeight+1)) 
-   || SQLITE_OK!=(rc = nodeRelease(pRtree, pParent))
-  ){
+  rc = nodeParentIndex(pRtree, pNode, &iCell);
+  if( rc==SQLITE_OK ){
+    pParent = pNode->pParent;
+    pNode->pParent = 0;
+    rc = deleteCell(pRtree, pParent, iCell, iHeight+1);
+  }
+  rc2 = nodeRelease(pRtree, pParent);
+  if( rc==SQLITE_OK ){
+    rc = rc2;
+  }
+  if( rc!=SQLITE_OK ){
     return rc;
   }
 
@@ -2096,8 +2410,9 @@ static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){
   return SQLITE_OK;
 }
 
-static void fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){
+static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){
   RtreeNode *pParent = pNode->pParent;
+  int rc = SQLITE_OK; 
   if( pParent ){
     int ii; 
     int nCell = NCELL(pNode);
@@ -2109,10 +2424,13 @@ static void fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){
       cellUnion(pRtree, &box, &cell);
     }
     box.iRowid = pNode->iNode;
-    ii = nodeParentIndex(pRtree, pNode);
-    nodeOverwriteCell(pRtree, pParent, &box, ii);
-    fixBoundingBox(pRtree, pParent);
+    rc = nodeParentIndex(pRtree, pNode, &ii);
+    if( rc==SQLITE_OK ){
+      nodeOverwriteCell(pRtree, pParent, &box, ii);
+      rc = fixBoundingBox(pRtree, pParent);
+    }
   }
+  return rc;
 }
 
 /*
@@ -2120,6 +2438,7 @@ static void fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){
 ** cell, adjust the r-tree data structure if required.
 */
 static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){
+  RtreeNode *pParent;
   int rc;
 
   if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){
@@ -2136,14 +2455,13 @@ static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){
   ** cell in the parent node so that it tightly contains the updated
   ** node.
   */
-  if( pNode->iNode!=1 ){
-    RtreeNode *pParent = pNode->pParent;
-    if( (pParent->iNode!=1 || NCELL(pParent)!=1) 
-     && (NCELL(pNode)<RTREE_MINCELLS(pRtree))
-    ){
+  pParent = pNode->pParent;
+  assert( pParent || pNode->iNode==1 );
+  if( pParent ){
+    if( NCELL(pNode)<RTREE_MINCELLS(pRtree) ){
       rc = removeNode(pRtree, pNode, iHeight);
     }else{
-      fixBoundingBox(pRtree, pNode);
+      rc = fixBoundingBox(pRtree, pNode);
     }
   }
 
@@ -2226,7 +2544,7 @@ static int Reinsert(
     }
   }
   if( rc==SQLITE_OK ){
-    fixBoundingBox(pRtree, pNode);
+    rc = fixBoundingBox(pRtree, pNode);
   }
   for(; rc==SQLITE_OK && ii<nCell; ii++){
     /* Find a node to store this cell in. pNode->iNode currently contains
@@ -2280,11 +2598,13 @@ static int rtreeInsertCell(
     rc = SplitNode(pRtree, pNode, pCell, iHeight);
 #endif
   }else{
-    AdjustTree(pRtree, pNode, pCell);
-    if( iHeight==0 ){
-      rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode);
-    }else{
-      rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode);
+    rc = AdjustTree(pRtree, pNode, pCell);
+    if( rc==SQLITE_OK ){
+      if( iHeight==0 ){
+        rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode);
+      }else{
+        rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode);
+      }
     }
   }
   return rc;
@@ -2354,7 +2674,6 @@ static int rtreeUpdate(
   rtreeReference(pRtree);
 
   assert(nData>=1);
-  assert(hashIsEmpty(pRtree));
 
   /* If azData[0] is not an SQL NULL value, it is the rowid of a
   ** record to delete from the r-tree table. The following block does
@@ -2380,8 +2699,10 @@ static int rtreeUpdate(
     /* Delete the cell in question from the leaf node. */
     if( rc==SQLITE_OK ){
       int rc2;
-      iCell = nodeRowidIndex(pRtree, pLeaf, iDelete);
-      rc = deleteCell(pRtree, pLeaf, iCell, 0);
+      rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell);
+      if( rc==SQLITE_OK ){
+        rc = deleteCell(pRtree, pLeaf, iCell, 0);
+      }
       rc2 = nodeRelease(pRtree, pLeaf);
       if( rc==SQLITE_OK ){
         rc = rc2;
@@ -2403,19 +2724,20 @@ static int rtreeUpdate(
     ** the root node (the operation that Gutman's paper says to perform 
     ** in this scenario).
     */
-    if( rc==SQLITE_OK && pRtree->iDepth>0 ){
-      if( rc==SQLITE_OK && NCELL(pRoot)==1 ){
-        RtreeNode *pChild;
-        i64 iChild = nodeGetRowid(pRtree, pRoot, 0);
-        rc = nodeAcquire(pRtree, iChild, pRoot, &pChild);
-        if( rc==SQLITE_OK ){
-          rc = removeNode(pRtree, pChild, pRtree->iDepth-1);
-        }
-        if( rc==SQLITE_OK ){
-          pRtree->iDepth--;
-          writeInt16(pRoot->zData, pRtree->iDepth);
-          pRoot->isDirty = 1;
-        }
+    if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){
+      int rc2;
+      RtreeNode *pChild;
+      i64 iChild = nodeGetRowid(pRtree, pRoot, 0);
+      rc = nodeAcquire(pRtree, iChild, pRoot, &pChild);
+      if( rc==SQLITE_OK ){
+        rc = removeNode(pRtree, pChild, pRtree->iDepth-1);
+      }
+      rc2 = nodeRelease(pRtree, pChild);
+      if( rc==SQLITE_OK ) rc = rc2;
+      if( rc==SQLITE_OK ){
+        pRtree->iDepth--;
+        writeInt16(pRoot->zData, pRtree->iDepth);
+        pRoot->isDirty = 1;
       }
     }
 
@@ -2481,6 +2803,7 @@ static int rtreeUpdate(
       }
       rc = sqlite3_reset(pRtree->pReadRowid);
     }
+    *pRowid = cell.iRowid;
 
     if( rc==SQLITE_OK ){
       rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf);
@@ -2618,31 +2941,69 @@ static int rtreeSqlInit(
 }
 
 /*
-** This routine queries database handle db for the page-size used by
-** database zDb. If successful, the page-size in bytes is written to
-** *piPageSize and SQLITE_OK returned. Otherwise, and an SQLite error 
-** code is returned.
+** The second argument to this function contains the text of an SQL statement
+** that returns a single integer value. The statement is compiled and executed
+** using database connection db. If successful, the integer value returned
+** is written to *piVal and SQLITE_OK returned. Otherwise, an SQLite error
+** code is returned and the value of *piVal after returning is not defined.
 */
-static int getPageSize(sqlite3 *db, const char *zDb, int *piPageSize){
+static int getIntFromStmt(sqlite3 *db, const char *zSql, int *piVal){
   int rc = SQLITE_NOMEM;
-  char *zSql;
-  sqlite3_stmt *pStmt = 0;
-
-  zSql = sqlite3_mprintf("PRAGMA %Q.page_size", zDb);
-  if( !zSql ){
-    return SQLITE_NOMEM;
+  if( zSql ){
+    sqlite3_stmt *pStmt = 0;
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+    if( rc==SQLITE_OK ){
+      if( SQLITE_ROW==sqlite3_step(pStmt) ){
+        *piVal = sqlite3_column_int(pStmt, 0);
+      }
+      rc = sqlite3_finalize(pStmt);
+    }
   }
+  return rc;
+}
 
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-  if( rc!=SQLITE_OK ){
-    return rc;
+/*
+** This function is called from within the xConnect() or xCreate() method to
+** determine the node-size used by the rtree table being created or connected
+** to. If successful, pRtree->iNodeSize is populated and SQLITE_OK returned.
+** Otherwise, an SQLite error code is returned.
+**
+** If this function is being called as part of an xConnect(), then the rtree
+** table already exists. In this case the node-size is determined by inspecting
+** the root node of the tree.
+**
+** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. 
+** This ensures that each node is stored on a single database page. If the 
+** database page-size is so large that more than RTREE_MAXCELLS entries 
+** would fit in a single node, use a smaller node-size.
+*/
+static int getNodeSize(
+  sqlite3 *db,                    /* Database handle */
+  Rtree *pRtree,                  /* Rtree handle */
+  int isCreate                    /* True for xCreate, false for xConnect */
+){
+  int rc;
+  char *zSql;
+  if( isCreate ){
+    int iPageSize;
+    zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb);
+    rc = getIntFromStmt(db, zSql, &iPageSize);
+    if( rc==SQLITE_OK ){
+      pRtree->iNodeSize = iPageSize-64;
+      if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)<pRtree->iNodeSize ){
+        pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS;
+      }
+    }
+  }else{
+    zSql = sqlite3_mprintf(
+        "SELECT length(data) FROM '%q'.'%q_node' WHERE nodeno = 1",
+        pRtree->zDb, pRtree->zName
+    );
+    rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize);
   }
 
-  if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    *piPageSize = sqlite3_column_int(pStmt, 0);
-  }
-  return sqlite3_finalize(pStmt);
+  sqlite3_free(zSql);
+  return rc;
 }
 
 /* 
@@ -2663,11 +3024,10 @@ static int rtreeInit(
   int isCreate                        /* True for xCreate, false for xConnect */
 ){
   int rc = SQLITE_OK;
-  int iPageSize = 0;
   Rtree *pRtree;
   int nDb;              /* Length of string argv[1] */
   int nName;            /* Length of string argv[2] */
-  int eCoordType = (int)pAux;
+  int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32);
 
   const char *aErrMsg[] = {
     0,                                                    /* 0 */
@@ -2682,11 +3042,6 @@ static int rtreeInit(
     return SQLITE_ERROR;
   }
 
-  rc = getPageSize(db, argv[1], &iPageSize);
-  if( rc!=SQLITE_OK ){
-    return rc;
-  }
-
   /* Allocate the sqlite3_vtab structure */
   nDb = strlen(argv[1]);
   nName = strlen(argv[2]);
@@ -2705,44 +3060,37 @@ static int rtreeInit(
   memcpy(pRtree->zDb, argv[1], nDb);
   memcpy(pRtree->zName, argv[2], nName);
 
-  /* Figure out the node size to use. By default, use 64 bytes less than
-  ** the database page-size. This ensures that each node is stored on
-  ** a single database page.
-  **
-  ** If the databasd page-size is so large that more than RTREE_MAXCELLS
-  ** entries would fit in a single node, use a smaller node-size.
-  */
-  pRtree->iNodeSize = iPageSize-64;
-  if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)<pRtree->iNodeSize ){
-    pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS;
-  }
+  /* Figure out the node size to use. */
+  rc = getNodeSize(db, pRtree, isCreate);
 
   /* Create/Connect to the underlying relational database schema. If
   ** that is successful, call sqlite3_declare_vtab() to configure
   ** the r-tree table schema.
   */
-  if( (rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate)) ){
-    *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
-  }else{
-    char *zSql = sqlite3_mprintf("CREATE TABLE x(%s", argv[3]);
-    char *zTmp;
-    int ii;
-    for(ii=4; zSql && ii<argc; ii++){
-      zTmp = zSql;
-      zSql = sqlite3_mprintf("%s, %s", zTmp, argv[ii]);
-      sqlite3_free(zTmp);
-    }
-    if( zSql ){
-      zTmp = zSql;
-      zSql = sqlite3_mprintf("%s);", zTmp);
-      sqlite3_free(zTmp);
-    }
-    if( !zSql ){
-      rc = SQLITE_NOMEM;
-    }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){
+  if( rc==SQLITE_OK ){
+    if( (rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate)) ){
       *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+    }else{
+      char *zSql = sqlite3_mprintf("CREATE TABLE x(%s", argv[3]);
+      char *zTmp;
+      int ii;
+      for(ii=4; zSql && ii<argc; ii++){
+        zTmp = zSql;
+        zSql = sqlite3_mprintf("%s, %s", zTmp, argv[ii]);
+        sqlite3_free(zTmp);
+      }
+      if( zSql ){
+        zTmp = zSql;
+        zSql = sqlite3_mprintf("%s);", zTmp);
+        sqlite3_free(zTmp);
+      }
+      if( !zSql ){
+        rc = SQLITE_NOMEM;
+      }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){
+        *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+      }
+      sqlite3_free(zSql);
     }
-    sqlite3_free(zSql);
   }
 
   if( rc==SQLITE_OK ){
@@ -2825,12 +3173,10 @@ static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){
 ** function "rtreenode".
 */
 int sqlite3RtreeInit(sqlite3 *db){
-  int rc = SQLITE_OK;
+  const int utf8 = SQLITE_UTF8;
+  int rc;
 
-  if( rc==SQLITE_OK ){
-    int utf8 = SQLITE_UTF8;
-    rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0);
-  }
+  rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0);
   if( rc==SQLITE_OK ){
     int utf8 = SQLITE_UTF8;
     rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0);
@@ -2847,6 +3193,70 @@ int sqlite3RtreeInit(sqlite3 *db){
   return rc;
 }
 
+/*
+** A version of sqlite3_free() that can be used as a callback. This is used
+** in two places - as the destructor for the blob value returned by the
+** invocation of a geometry function, and as the destructor for the geometry
+** functions themselves.
+*/
+static void doSqlite3Free(void *p){
+  sqlite3_free(p);
+}
+
+/*
+** Each call to sqlite3_rtree_geometry_callback() creates an ordinary SQLite
+** scalar user function. This C function is the callback used for all such
+** registered SQL functions.
+**
+** The scalar user functions return a blob that is interpreted by r-tree
+** table MATCH operators.
+*/
+static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){
+  RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx);
+  RtreeMatchArg *pBlob;
+  int nBlob;
+
+  nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(double);
+  pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob);
+  if( !pBlob ){
+    sqlite3_result_error_nomem(ctx);
+  }else{
+    int i;
+    pBlob->magic = RTREE_GEOMETRY_MAGIC;
+    pBlob->xGeom = pGeomCtx->xGeom;
+    pBlob->pContext = pGeomCtx->pContext;
+    pBlob->nParam = nArg;
+    for(i=0; i<nArg; i++){
+      pBlob->aParam[i] = sqlite3_value_double(aArg[i]);
+    }
+    sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free);
+  }
+}
+
+/*
+** Register a new geometry function for use with the r-tree MATCH operator.
+*/
+int sqlite3_rtree_geometry_callback(
+  sqlite3 *db,
+  const char *zGeom,
+  int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *),
+  void *pContext
+){
+  RtreeGeomCallback *pGeomCtx;      /* Context object for new user-function */
+
+  /* Allocate and populate the context object. */
+  pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback));
+  if( !pGeomCtx ) return SQLITE_NOMEM;
+  pGeomCtx->xGeom = xGeom;
+  pGeomCtx->pContext = pContext;
+
+  /* Create the new user-function. Register a destructor function to delete
+  ** the context object when it is no longer required.  */
+  return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, 
+      (void *)pGeomCtx, geomCallback, 0, 0, doSqlite3Free
+  );
+}
+
 #if !SQLITE_CORE
 int sqlite3_extension_init(
   sqlite3 *db,
diff --git a/third_party/sqlite/src/ext/rtree/rtree1.test b/third_party/sqlite/src/ext/rtree/rtree1.test
index 8c1675b..fe5fa0a 100755..100644
--- a/third_party/sqlite/src/ext/rtree/rtree1.test
+++ b/third_party/sqlite/src/ext/rtree/rtree1.test
@@ -11,11 +11,9 @@
 #
 # The focus of this file is testing the r-tree extension.
 #
-# $Id: rtree1.test,v 1.7 2009/07/17 16:54:48 danielk1977 Exp $
-#
 
 if {![info exists testdir]} {
-  set testdir [file join [file dirname $argv0] .. .. test]
+  set testdir [file join [file dirname [info script]] .. .. test]
 }
 source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
@@ -401,4 +399,21 @@ do_test rtree-10.1 {
   catchsql { CREATE VIRTUAL TABLE t7 USING rtree(index, x1, y1, x2, y2) }
 } {1 {near "index": syntax error}}
 
+#-------------------------------------------------------------------------
+# Test last_insert_rowid().
+# 
+do_test rtree-11.1 {
+  execsql {
+    CREATE VIRTUAL TABLE t8 USING rtree(idx, x1, x2, y1, y2);
+    INSERT INTO t8 VALUES(1, 1.0, 1.0, 2.0, 2.0);
+    SELECT last_insert_rowid();
+  }
+} {1}
+do_test rtree-11.2 {
+  execsql {
+    INSERT INTO t8 VALUES(NULL, 1.0, 1.0, 2.0, 2.0);
+    SELECT last_insert_rowid();
+  }
+} {2}
+
 finish_test
diff --git a/third_party/sqlite/src/ext/rtree/rtree2.test b/third_party/sqlite/src/ext/rtree/rtree2.test
index 7e38c8f..f5d15cc 100755..100644
--- a/third_party/sqlite/src/ext/rtree/rtree2.test
+++ b/third_party/sqlite/src/ext/rtree/rtree2.test
@@ -11,11 +11,9 @@
 #
 # The focus of this file is testing the r-tree extension.
 #
-# $Id: rtree2.test,v 1.4 2008/07/14 15:37:01 danielk1977 Exp $
-#
 
 if {![info exists testdir]} {
-  set testdir [file join [file dirname $argv0] .. .. test]
+  set testdir [file join [file dirname [info script]] .. .. test]
 } 
 source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
@@ -29,7 +27,7 @@ set ::NROW   1000
 set ::NDEL     10
 set ::NSELECT 100
 
-if {[info exists ISQUICK] && $ISQUICK} {
+if {[info exists G(isquick)] && $G(isquick)} {
   set ::NROW 100
   set ::NSELECT 10
 }
diff --git a/third_party/sqlite/src/ext/rtree/rtree3.test b/third_party/sqlite/src/ext/rtree/rtree3.test
index b83ceeb4..fea5513 100755..100644
--- a/third_party/sqlite/src/ext/rtree/rtree3.test
+++ b/third_party/sqlite/src/ext/rtree/rtree3.test
@@ -12,56 +12,95 @@
 # The focus of this file is testing that the r-tree correctly handles
 # out-of-memory conditions.
 #
-# $Id: rtree3.test,v 1.2 2008/06/23 15:55:52 danielk1977 Exp $
-#
 
 if {![info exists testdir]} {
-  set testdir [file join [file dirname $argv0] .. .. test]
+  set testdir [file join [file dirname [info script]] .. .. test]
 } 
 source $testdir/tester.tcl
-
+source $testdir/malloc_common.tcl
 ifcapable !rtree {
   finish_test
   return
 }
 
-# Only run these tests if memory debugging is turned on.
+# Test summary:
 #
-source $testdir/malloc_common.tcl
-if {!$MEMDEBUG} {
-   puts "Skipping malloc tests: not compiled with -DSQLITE_MEMDEBUG..."
-   finish_test
-   return
+#   rtree3-1: Test OOM in simple CREATE TABLE, INSERT, DELETE and SELECT 
+#             commands on an almost empty table.
+#
+#   rtree3-2: Test OOM in a DROP TABLE command.
+#
+#   rtree3-3a: Test OOM during a transaction to insert 100 pseudo-random rows.
+#
+#   rtree3-3b: Test OOM during a transaction deleting all entries in the
+#              database constructed in [rtree3-3a] in pseudo-random order.
+#
+#   rtree3-4a: OOM during "SELECT count(*) FROM ..." on a big table.
+#
+#   rtree3-4b: OOM while deleting rows from a big table.
+#
+#   rtree3-5: Test OOM while inserting rows into a big table.
+#
+#   rtree3-6: Test OOM while deleting all rows of a table, one at a time.
+#
+#   rtree3-7: OOM during an ALTER TABLE RENAME TABLE command.
+#
+#   rtree3-8: Test OOM while registering the r-tree module with sqlite.
+#
+
+do_faultsim_test rtree3-1 -faults oom* -prep {
+  faultsim_delete_and_reopen
+} -body {
+  execsql {
+    BEGIN TRANSACTION;
+    CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
+    INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
+    INSERT INTO rt VALUES(NULL, 13, 15, 17, 19);
+    DELETE FROM rt WHERE ii = 1;
+    SELECT * FROM rt;
+    SELECT ii FROM rt WHERE ii = 2;
+    COMMIT;
+  }
 }
 
-do_malloc_test rtree3-1 -sqlbody {
-  BEGIN TRANSACTION;
-  CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
-  INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
-  INSERT INTO rt VALUES(NULL, 13, 15, 17, 19);
-  DELETE FROM rt WHERE ii = 1;
-  SELECT * FROM rt;
-  SELECT ii FROM rt WHERE ii = 2;
-  COMMIT;
-} 
-do_malloc_test rtree3-2 -sqlprep {
-  CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
-  INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
-} -sqlbody {
-  DROP TABLE rt;
-} 
+do_test rtree3-2.prep {
+  faultsim_delete_and_reopen
+  execsql {
+    CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
+    INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
+  }
+  faultsim_save_and_close
+} {}
+do_faultsim_test rtree3-2 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { DROP TABLE rt } 
+}
 
+do_malloc_test rtree3-3.prep {
+  faultsim_delete_and_reopen
+  execsql {
+    CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
+    INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
+  }
+  faultsim_save_and_close
+} {}
 
-do_malloc_test rtree3-3 -sqlprep {
-  CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
-  INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
-} -tclbody {
+do_faultsim_test rtree3-3a -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
   db eval BEGIN
   for {set ii 0} {$ii < 100} {incr ii} {
     set f [expr rand()]
     db eval {INSERT INTO rt VALUES(NULL, $f*10.0, $f*10.0, $f*15.0, $f*15.0)}
   }
   db eval COMMIT
+}
+faultsim_save_and_close
+
+do_faultsim_test rtree3-3b -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
   db eval BEGIN
   for {set ii 0} {$ii < 100} {incr ii} {
     set f [expr rand()]
@@ -70,5 +109,129 @@ do_malloc_test rtree3-3 -sqlprep {
   db eval COMMIT
 } 
 
-finish_test
+do_test rtree3-4.prep {
+  faultsim_delete_and_reopen
+  execsql {
+    BEGIN;
+    PRAGMA page_size = 512;
+    CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
+  }
+  for {set i 0} {$i < 1500} {incr i} {
+    execsql { INSERT INTO rt VALUES($i, $i, $i+1, $i, $i+1) }
+  }
+  execsql { COMMIT }
+  faultsim_save_and_close
+} {}
+
+do_faultsim_test rtree3-4a -faults oom-* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  db eval { SELECT count(*) FROM rt }
+} -test {
+  faultsim_test_result {0 1500}
+}
 
+do_faultsim_test rtree3-4b -faults oom-transient -prep {
+  faultsim_restore_and_reopen
+} -body {
+  db eval { DELETE FROM rt WHERE ii BETWEEN 1 AND 100 }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_test rtree3-5.prep {
+  faultsim_delete_and_reopen
+  execsql {
+    BEGIN;
+    PRAGMA page_size = 512;
+    CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
+  }
+  for {set i 0} {$i < 100} {incr i} {
+    execsql { INSERT INTO rt VALUES($i, $i, $i+1, $i, $i+1) }
+  }
+  execsql { COMMIT }
+  faultsim_save_and_close
+} {}
+do_faultsim_test rtree3-5 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  for {set i 100} {$i < 110} {incr i} {
+    execsql { INSERT INTO rt VALUES($i, $i, $i+1, $i, $i+1) }
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_test rtree3-6.prep {
+  faultsim_delete_and_reopen
+  execsql {
+    BEGIN;
+    PRAGMA page_size = 512;
+    CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
+  }
+  for {set i 0} {$i < 50} {incr i} {
+    execsql { INSERT INTO rt VALUES($i, $i, $i+1, $i, $i+1) }
+  }
+  execsql { COMMIT }
+  faultsim_save_and_close
+} {}
+do_faultsim_test rtree3-6 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql BEGIN
+  for {set i 0} {$i < 50} {incr i} {
+    execsql { DELETE FROM rt WHERE ii=$i }
+  }
+  execsql COMMIT
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_test rtree3-7.prep {
+  faultsim_delete_and_reopen
+  execsql { CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2) }
+  faultsim_save_and_close
+} {}
+do_faultsim_test rtree3-7 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { ALTER TABLE rt RENAME TO rt2 }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_faultsim_test rtree3-8 -faults oom-* -prep {
+  catch { db close }
+} -body {
+  sqlite3 db test.db
+} 
+
+do_faultsim_test rtree3-9 -faults oom-* -prep {
+  sqlite3 db :memory:
+} -body {
+  set rc [register_cube_geom db]
+  if {$rc != "SQLITE_OK"} { error $rc }
+} -test {
+  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
+}
+
+do_test rtree3-10.prep {
+  faultsim_delete_and_reopen
+  execsql { 
+    CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2, z1, z2);
+    INSERT INTO rt VALUES(1,  10, 10, 10, 11, 11, 11);
+    INSERT INTO rt VALUES(2,  5, 6, 6, 7, 7, 8);
+  }
+  faultsim_save_and_close
+} {}
+do_faultsim_test rtree3-10 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+  register_cube_geom db
+  execsql { SELECT * FROM rt }
+} -body {
+  execsql { SELECT ii FROM rt WHERE ii MATCH cube(4.5, 5.5, 6.5, 1, 1, 1) }
+} -test {
+  faultsim_test_result {0 2}
+}
+
+finish_test
diff --git a/third_party/sqlite/src/ext/rtree/rtree4.test b/third_party/sqlite/src/ext/rtree/rtree4.test
index d73e7a6..708d335 100755..100644
--- a/third_party/sqlite/src/ext/rtree/rtree4.test
+++ b/third_party/sqlite/src/ext/rtree/rtree4.test
@@ -11,11 +11,9 @@
 #
 # Randomized test cases for the rtree extension.
 #
-# $Id: rtree4.test,v 1.3 2008/06/23 15:55:52 danielk1977 Exp $
-#
 
 if {![info exists testdir]} {
-  set testdir [file join [file dirname $argv0] .. .. test]
+  set testdir [file join [file dirname [info script]] .. .. test]
 } 
 source $testdir/tester.tcl
 
@@ -25,7 +23,7 @@ ifcapable !rtree {
 }
 
 set ::NROW 2500
-if {[info exists ISQUICK] && $ISQUICK} {
+if {[info exists G(isquick)] && $G(isquick)} {
   set ::NROW 250
 }
 
@@ -95,7 +93,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
   for {set i 1} {$i<$::NROW} {incr i} {
     # Do a random insert
     #
-    do_test rtree-$nDim.2.$i.1 {
+    do_test rtree4-$nDim.2.$i.1 {
       set vlist {}
       for {set j 0} {$j<$nDim} {incr j} {
         set mn [rand 10000]
@@ -115,7 +113,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
       lappend where mn$j>=$mn mx$j<=$mx
     }
     set where "WHERE [join $where { AND }]"
-    do_test rtree-$nDim.2.$i.2 {
+    do_test rtree4-$nDim.2.$i.2 {
       list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
     } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
 
@@ -128,7 +126,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
       lappend where mx$j>=$mn mn$j<=$mx
     }
     set where "WHERE [join $where { AND }]"
-    do_test rtree-$nDim.2.$i.3 {
+    do_test rtree4-$nDim.2.$i.3 {
       list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
     } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
 
@@ -145,7 +143,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
       lappend where mn$j>=$mn mx$j<=$mx
     }
     set where "WHERE [join $where { AND }]"
-    do_test rtree-$nDim.2.$i.3 {
+    do_test rtree4-$nDim.2.$i.3 {
       list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
     } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
 
@@ -162,7 +160,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
       lappend where mx$j>$mn mn$j<$mx
     }
     set where "WHERE [join $where { AND }]"
-    do_test rtree-$nDim.2.$i.4 {
+    do_test rtree4-$nDim.2.$i.4 {
       list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
     } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
 
@@ -178,7 +176,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
       lappend where mn$j>=-10000 mx$j<10000
     }
     set where "WHERE [join $where { AND }]"
-    do_test rtree-$nDim.2.$i.5 {
+    do_test rtree4-$nDim.2.$i.5 {
       list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
     } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
 
@@ -194,7 +192,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
       lappend where mx$j>-10000 mn$j<=10000
     }
     set where "WHERE [join $where { AND }]"
-    do_test rtree-$nDim.2.$i.6 {
+    do_test rtree4-$nDim.2.$i.6 {
       list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
     } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
 
@@ -210,7 +208,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
       lappend where mn$j>=$mn1 mn$j>$mn2 mx$j<$mx1 mx$j<=$mx2
     }
     set where "WHERE [join [scramble $where] { AND }]"
-    do_test rtree-$nDim.2.$i.7 {
+    do_test rtree4-$nDim.2.$i.7 {
       list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
     } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
 
@@ -226,7 +224,7 @@ for {set nDim 1} {$nDim<=5} {incr nDim} {
       lappend where mx$j>=$mn1 mx$j>$mn2 mn$j<$mx1 mn$j<=$mx2
     }
     set where "WHERE [join [scramble $where] { AND }]"
-    do_test rtree-$nDim.2.$i.8 {
+    do_test rtree4-$nDim.2.$i.8 {
       list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
     } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
   }
diff --git a/third_party/sqlite/src/ext/rtree/rtree5.test b/third_party/sqlite/src/ext/rtree/rtree5.test
index 4fa007f..ea2946f 100755..100644
--- a/third_party/sqlite/src/ext/rtree/rtree5.test
+++ b/third_party/sqlite/src/ext/rtree/rtree5.test
@@ -12,11 +12,9 @@
 # The focus of this file is testing the r-tree extension when it is
 # configured to store values as 32 bit integers.
 #
-# $Id: rtree5.test,v 1.1 2008/07/14 15:37:01 danielk1977 Exp $
-#
 
 if {![info exists testdir]} {
-  set testdir [file join [file dirname $argv0] .. .. test]
+  set testdir [file join [file dirname [info script]] .. .. test]
 } 
 source $testdir/tester.tcl
 
diff --git a/third_party/sqlite/src/ext/rtree/rtree6.test b/third_party/sqlite/src/ext/rtree/rtree6.test
index affa8fe..0a65c15 100644
--- a/third_party/sqlite/src/ext/rtree/rtree6.test
+++ b/third_party/sqlite/src/ext/rtree/rtree6.test
@@ -8,12 +8,11 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
-#
-# $Id: rtree6.test,v 1.1 2008/09/01 12:47:00 danielk1977 Exp $
+# 
 #
 
 if {![info exists testdir]} {
-  set testdir [file join [file dirname $argv0] .. .. test]
+  set testdir [file join [file dirname [info script]] .. .. test]
 } 
 source $testdir/tester.tcl
 
@@ -89,8 +88,8 @@ do_test rtree6.2.2 {
 do_test rtree6.2.3 {
   query_plan {SELECT * FROM t1,t2 WHERE k=ii}
 } [list \
-  {TABLE t2}                          \
-  {TABLE t1 VIRTUAL TABLE INDEX 1:}   \
+  {TABLE t1 VIRTUAL TABLE INDEX 2:}   \
+  {TABLE t2 USING PRIMARY KEY}        \
 ]
 
 do_test rtree6.2.4 {
@@ -103,9 +102,8 @@ do_test rtree6.2.4 {
 do_test rtree6.2.5 {
   query_plan {SELECT * FROM t1,t2 WHERE k=ii AND x1<v}
 } [list \
-  {TABLE t2}                              \
-  {TABLE t1 VIRTUAL TABLE INDEX 1:}   \
+  {TABLE t1 VIRTUAL TABLE INDEX 2:}   \
+  {TABLE t2 USING PRIMARY KEY}        \
 ]
 
 finish_test
-
diff --git a/third_party/sqlite/src/ext/rtree/rtree7.test b/third_party/sqlite/src/ext/rtree/rtree7.test
new file mode 100644
index 0000000..31dae0c
--- /dev/null
+++ b/third_party/sqlite/src/ext/rtree/rtree7.test
@@ -0,0 +1,58 @@
+# 2010 February 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# Test that nothing goes wrong if an rtree table is created, then the
+# database page-size is modified. At one point (3.6.22), this was causing
+# malfunctions.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+
+ifcapable !rtree||!vacuum {
+  finish_test
+  return
+}
+
+do_test rtree7-1.1 {
+  execsql {
+    PRAGMA page_size = 1024;
+    CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2, y1, y2);
+    INSERT INTO rt VALUES(1, 1, 2, 3, 4);
+  }
+} {}
+do_test rtree7-1.2 {
+  execsql { SELECT * FROM rt }
+} {1 1.0 2.0 3.0 4.0}
+do_test rtree7-1.3 {
+  execsql { 
+    PRAGMA page_size = 2048;
+    VACUUM;
+    SELECT * FROM rt;
+  }
+} {1 1.0 2.0 3.0 4.0}
+do_test rtree7-1.4 {
+  for {set i 2} {$i <= 51} {incr i} {
+    execsql { INSERT INTO rt VALUES($i, 1, 2, 3, 4) }
+  }
+  execsql { SELECT sum(x1), sum(x2), sum(y1), sum(y2) FROM rt }
+} {51.0 102.0 153.0 204.0}
+do_test rtree7-1.5 {
+  execsql { 
+    PRAGMA page_size = 512;
+    VACUUM;
+    SELECT sum(x1), sum(x2), sum(y1), sum(y2) FROM rt
+  }
+} {51.0 102.0 153.0 204.0}
+
+finish_test
diff --git a/third_party/sqlite/src/ext/rtree/rtree8.test b/third_party/sqlite/src/ext/rtree/rtree8.test
new file mode 100644
index 0000000..bf22cbf
--- /dev/null
+++ b/third_party/sqlite/src/ext/rtree/rtree8.test
@@ -0,0 +1,171 @@
+# 2010 February 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+ifcapable !rtree { finish_test ; return }
+
+#-------------------------------------------------------------------------
+# The following block of tests - rtree8-1.* - feature reading and writing
+# an r-tree table while there exist open cursors on it.
+#
+proc populate_t1 {n} {
+  execsql { DELETE FROM t1 }
+  for {set i 1} {$i <= $n} {incr i} {
+    execsql { INSERT INTO t1 VALUES($i, $i, $i+2) }
+  }
+}
+
+# A DELETE while a cursor is reading the table.
+#
+do_test rtree8-1.1.1 {
+  execsql { PRAGMA page_size = 512 }
+  execsql { CREATE VIRTUAL TABLE t1 USING rtree_i32(id, x1, x2) }
+  populate_t1 5
+} {}
+do_test rtree8-1.1.2 {
+  set res [list]
+  db eval { SELECT * FROM t1 } { 
+    lappend res $x1 $x2
+    if {$id==3} { db eval { DELETE FROM t1 WHERE id>3 } }
+  }
+  set res
+} {1 3 2 4 3 5}
+do_test rtree8-1.1.3 {
+  execsql { SELECT * FROM t1 }
+} {1 1 3 2 2 4 3 3 5}
+
+# Many SELECTs on the same small table.
+#
+proc nested_select {n} {
+  set ::max $n
+  db eval { SELECT * FROM t1 } {
+    if {$id == $n} { nested_select [expr $n+1] }
+  }
+  return $::max
+}
+do_test rtree8-1.2.1 { populate_t1 50  } {}
+do_test rtree8-1.2.2 { nested_select 1 } {51}
+
+# This test runs many SELECT queries simultaneously against a large 
+# table, causing a collision in the hash-table used to store r-tree 
+# nodes internally.
+#
+populate_t1 1500
+do_execsql_test rtree8-1.3.1 { SELECT max(nodeno) FROM t1_node } {164}
+do_test rtree8-1.3.2 {
+  set rowids [execsql {SELECT min(rowid) FROM t1_rowid GROUP BY nodeno}]
+  set stmt_list [list]
+  foreach row $rowids {
+    set stmt [sqlite3_prepare db "SELECT * FROM t1 WHERE id = $row" -1 tail]
+    sqlite3_step $stmt
+    lappend res_list [sqlite3_column_int $stmt 0]
+    lappend stmt_list $stmt 
+  }
+} {}
+do_test rtree8-1.3.3 { set res_list } $rowids
+do_execsql_test rtree8-1.3.4 { SELECT count(*) FROM t1 } {1500}
+do_test rtree8-1.3.5 { 
+  foreach stmt $stmt_list { sqlite3_finalize $stmt }
+} {}
+
+
+#-------------------------------------------------------------------------
+# The following block of tests - rtree8-2.* - test a couple of database
+# corruption cases. In this case things are not corrupted at the b-tree
+# level, but the contents of the various tables used internally by an
+# r-tree table are inconsistent.
+#
+populate_t1 50
+do_execsql_test rtree8-2.1.1 { SELECT max(nodeno) FROM t1_node } {5}
+do_execsql_test rtree8-2.1.2 { DELETE FROM t1_node } {}
+for {set i 1} {$i <= 50} {incr i} {
+  do_catchsql_test rtree8-2.1.3.$i { 
+    SELECT * FROM t1 WHERE id = $i 
+  } {1 {database disk image is malformed}}
+}
+do_catchsql_test rtree8-2.1.4 { 
+  SELECT * FROM t1
+} {1 {database disk image is malformed}}
+do_catchsql_test rtree8-2.1.5 { 
+  DELETE FROM t1
+} {1 {database disk image is malformed}}
+
+do_execsql_test rtree8-2.1.6 { 
+  DROP TABLE t1;
+  CREATE VIRTUAL TABLE t1 USING rtree_i32(id, x1, x2);
+} {}
+
+
+populate_t1 50
+do_execsql_test rtree8-2.2.1 {
+  DELETE FROM t1_parent
+} {}
+do_catchsql_test rtree8-2.2.2 {
+  DELETE FROM t1 WHERE id=25
+} {1 {database disk image is malformed}}
+do_execsql_test rtree8-2.2.3 { 
+  DROP TABLE t1;
+  CREATE VIRTUAL TABLE t1 USING rtree_i32(id, x1, x2);
+} {}
+
+
+#-------------------------------------------------------------------------
+# Test that trying to use the MATCH operator with the r-tree module does
+# not confuse it. 
+#
+populate_t1 10
+do_catchsql_test rtree8-3.1 { 
+  SELECT * FROM t1 WHERE x1 MATCH '1234'
+} {1 {SQL logic error or missing database}}
+
+#-------------------------------------------------------------------------
+# Test a couple of invalid arguments to rtreedepth().
+#
+do_catchsql_test rtree8-4.1 {
+  SELECT rtreedepth('hello world')
+} {1 {Invalid argument to rtreedepth()}}
+do_catchsql_test rtree8-4.2 {
+  SELECT rtreedepth(X'00')
+} {1 {Invalid argument to rtreedepth()}}
+
+
+#-------------------------------------------------------------------------
+# Delete half of a lopsided tree.
+#
+do_execsql_test rtree8-5.1 { 
+  CREATE VIRTUAL TABLE t2 USING rtree_i32(id, x1, x2) 
+} {}
+do_test rtree8-5.2 {
+  execsql BEGIN
+  for {set i 0} {$i < 100} {incr i} {
+    execsql { INSERT INTO t2 VALUES($i, 100, 101) }
+  }
+  for {set i 100} {$i < 200} {incr i} {
+    execsql { INSERT INTO t2 VALUES($i, 1000, 1001) }
+  }
+  execsql COMMIT
+} {}
+do_test rtree8-5.3 {
+  execsql BEGIN
+  for {set i 0} {$i < 200} {incr i} {
+    execsql { DELETE FROM t2 WHERE id = $i }
+  }
+  execsql COMMIT
+} {}
+
+
+finish_test
+
diff --git a/third_party/sqlite/src/ext/rtree/rtree9.test b/third_party/sqlite/src/ext/rtree/rtree9.test
new file mode 100644
index 0000000..ddee277
--- /dev/null
+++ b/third_party/sqlite/src/ext/rtree/rtree9.test
@@ -0,0 +1,125 @@
+# 2010 August 28
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file contains tests for the r-tree module. Specifically, it tests
+# that custom r-tree queries (geometry callbacks) work.
+# 
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+ifcapable !rtree { finish_test ; return }
+
+register_cube_geom db
+
+do_execsql_test rtree9-1.1 {
+  CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2, y1, y2, z1, z2);
+  INSERT INTO rt VALUES(1, 1, 2, 1, 2, 1, 2);
+} {}
+do_execsql_test rtree9-1.2 {
+  SELECT * FROM rt WHERE id MATCH cube(0, 0, 0, 2, 2, 2);
+} {1 1.0 2.0 1.0 2.0 1.0 2.0}
+do_execsql_test rtree9-1.3 {
+  SELECT * FROM rt WHERE id MATCH cube(3, 3, 3, 2, 2, 2);
+} {}
+do_execsql_test rtree9-1.4 {
+  DELETE FROM rt;
+} {}
+
+
+for {set i 0} {$i < 1000} {incr i} {
+  set x [expr $i%10]
+  set y [expr ($i/10)%10]
+  set z [expr ($i/100)%10]
+  execsql { INSERT INTO rt VALUES($i, $x, $x+1, $y, $y+1, $z, $z+1) }
+}
+do_execsql_test rtree9-2.1 {
+  SELECT id FROM rt WHERE id MATCH cube(2.5, 2.5, 2.5, 1, 1, 1) ORDER BY id;
+} {222 223 232 233 322 323 332 333}
+do_execsql_test rtree9-2.2 {
+  SELECT id FROM rt WHERE id MATCH cube(5.5, 5.5, 5.5, 1, 1, 1) ORDER BY id;
+} {555 556 565 566 655 656 665 666}
+
+
+do_execsql_test rtree9-3.1 {
+  CREATE VIRTUAL TABLE rt32 USING rtree_i32(id, x1, x2, y1, y2, z1, z2);
+} {} 
+for {set i 0} {$i < 1000} {incr i} {
+  set x [expr $i%10]
+  set y [expr ($i/10)%10]
+  set z [expr ($i/100)%10]
+  execsql { INSERT INTO rt32 VALUES($i, $x, $x+1, $y, $y+1, $z, $z+1) }
+}
+do_execsql_test rtree9-3.2 {
+  SELECT id FROM rt32 WHERE id MATCH cube(3, 3, 3, 1, 1, 1) ORDER BY id;
+} {222 223 224 232 233 234 242 243 244 322 323 324 332 333 334 342 343 344 422 423 424 432 433 434 442 443 444}
+do_execsql_test rtree9-3.3 {
+  SELECT id FROM rt32 WHERE id MATCH cube(5.5, 5.5, 5.5, 1, 1, 1) ORDER BY id;
+} {555 556 565 566 655 656 665 666}
+
+
+do_catchsql_test rtree9-4.1 {
+  SELECT id FROM rt32 WHERE id MATCH cube(5.5, 5.5, 1, 1, 1) ORDER BY id;
+} {1 {SQL logic error or missing database}}
+for {set x 2} {$x<200} {incr x 2} {
+  do_catchsql_test rtree9-4.2.[expr $x/2] {
+    SELECT id FROM rt WHERE id MATCH randomblob($x)
+  } {1 {SQL logic error or missing database}}
+}
+do_catchsql_test rtree9-4.3 {
+  SELECT id FROM rt WHERE id MATCH CAST( 
+    (cube(5.5, 5.5, 5.5, 1, 1, 1) || X'1234567812345678') AS blob 
+  )
+} {1 {SQL logic error or missing database}}
+
+
+#-------------------------------------------------------------------------
+# Test the example 2d "circle" geometry callback.
+#
+register_circle_geom db
+
+breakpoint
+do_execsql_test rtree9-5.1 {
+  CREATE VIRTUAL TABLE rt2 USING rtree(id, xmin, xmax, ymin, ymax);
+
+  INSERT INTO rt2 VALUES(1,    1,   2,  1,  2);
+  INSERT INTO rt2 VALUES(2,    1,   2, -2, -1);
+  INSERT INTO rt2 VALUES(3,    -2, -1, -2, -1);
+  INSERT INTO rt2 VALUES(4,    -2, -1,  1,  2);
+
+  INSERT INTO rt2 VALUES(5,    2,   3,  2,  3);
+  INSERT INTO rt2 VALUES(6,    2,   3, -3, -2);
+  INSERT INTO rt2 VALUES(7,    -3, -2, -3, -2);
+  INSERT INTO rt2 VALUES(8,    -3, -2,  2,  3);
+
+  INSERT INTO rt2 VALUES(9,    1.8,   3,  1.8,  3);
+  INSERT INTO rt2 VALUES(10,   1.8,   3, -3, -1.8);
+  INSERT INTO rt2 VALUES(11,   -3, -1.8, -3, -1.8);
+  INSERT INTO rt2 VALUES(12,   -3, -1.8,  1.8,  3);
+
+  INSERT INTO rt2 VALUES(13,   -15, 15, 1.8, 2.2);
+  INSERT INTO rt2 VALUES(14,   -15, 15, -2.2, -1.8);
+  INSERT INTO rt2 VALUES(15,   1.8, 2.2, -15, 15);
+  INSERT INTO rt2 VALUES(16,   -2.2, -1.8, -15, 15);
+
+  INSERT INTO rt2 VALUES(17,   -100, 100, -100, 100);
+} {}
+
+do_execsql_test rtree9-5.2 {
+  SELECT id FROM rt2 WHERE id MATCH circle(0.0, 0.0, 2.0);
+} {1 2 3 4 13 14 15 16 17}
+
+do_execsql_test rtree9-5.3 {
+  UPDATE rt2 SET xmin=xmin+5, ymin=ymin+5, xmax=xmax+5, ymax=ymax+5;
+  SELECT id FROM rt2 WHERE id MATCH circle(5.0, 5.0, 2.0);
+} {1 2 3 4 13 14 15 16 17}
+
+finish_test
diff --git a/third_party/sqlite/src/ext/rtree/rtreeA.test b/third_party/sqlite/src/ext/rtree/rtreeA.test
new file mode 100644
index 0000000..e377b01
--- /dev/null
+++ b/third_party/sqlite/src/ext/rtree/rtreeA.test
@@ -0,0 +1,220 @@
+# 2010 September 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file contains tests for the r-tree module. Specifically, it tests
+# that corrupt or inconsistent databases do not cause crashes in the r-tree
+# module.
+# 
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+} 
+source $testdir/tester.tcl
+ifcapable !rtree { finish_test ; return }
+
+proc create_t1 {} {
+  db close
+  forcedelete test.db
+  sqlite3 db test.db
+  execsql {
+    PRAGMA page_size = 1024;
+    CREATE VIRTUAL TABLE t1 USING rtree(id, x1, x2, y1, y2);
+  }
+}
+proc populate_t1 {} {
+  execsql BEGIN
+  for {set i 0} {$i < 500} {incr i} {
+    set x2 [expr $i+5]
+    set y2 [expr $i+5]
+    execsql { INSERT INTO t1 VALUES($i, $i, $x2, $i, $y2) }
+  }
+  execsql COMMIT
+}
+
+proc truncate_node {nodeno nTrunc} {
+  set blob [db one {SELECT data FROM t1_node WHERE nodeno=$nodeno}]
+  if {$nTrunc<0} {set nTrunc "end-$nTrunc"}
+  set blob [string range $blob 0 $nTrunc]
+  db eval { UPDATE t1_node SET data = $blob WHERE nodeno=$nodeno }
+}
+
+proc set_tree_depth {tbl {newvalue ""}} {
+  set blob [db one "SELECT data FROM ${tbl}_node WHERE nodeno=1"]
+
+  if {$newvalue == ""} {
+    binary scan $blob Su oldvalue
+    return $oldvalue
+  }
+
+  set blob [binary format Sua* $newvalue [string range $blob 2 end]]
+  db eval "UPDATE ${tbl}_node SET data = \$blob WHERE nodeno=1"
+  return [set_tree_depth $tbl]
+}
+
+proc set_entry_count {tbl nodeno {newvalue ""}} {
+  set blob [db one "SELECT data FROM ${tbl}_node WHERE nodeno=$nodeno"]
+
+  if {$newvalue == ""} {
+    binary scan [string range $blob 2 end] Su oldvalue
+    return $oldvalue
+  }
+
+  set blob [binary format a*Sua* \
+    [string range $blob 0 1] $newvalue [string range $blob 4 end]
+  ]
+  db eval "UPDATE ${tbl}_node SET data = \$blob WHERE nodeno=$nodeno"
+  return [set_entry_count $tbl $nodeno]
+}
+
+
+proc do_corruption_tests {prefix args} {
+  set testarray [lindex $args end]
+  set errormsg {database disk image is malformed}
+
+  foreach {z value} [lrange $args 0 end-1] {
+    set n [string length $z]
+    if {$n>=2 && [string equal -length $n $z "-error"]} {
+      set errormsg $value
+    }
+  }
+
+  foreach {tn sql} $testarray {
+    do_catchsql_test $prefix.$tn $sql [list 1 $errormsg]
+  }
+}
+
+#-------------------------------------------------------------------------
+# Test the libraries response if the %_node table is completely empty
+# (i.e. the root node is missing), or has been removed from the database
+# entirely.
+#
+create_t1
+populate_t1
+do_execsql_test rtreeA-1.0 {
+  DELETE FROM t1_node;
+} {}
+
+do_corruption_tests rtreeA-1.1 {
+  1   "SELECT * FROM t1"
+  2   "SELECT * FROM t1 WHERE rowid=5"
+  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
+  4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
+}
+
+do_execsql_test  rtreeA-1.2.0 { DROP TABLE t1_node } {}
+do_corruption_tests rtreeA-1.2 -error "SQL logic error or missing database" {
+  1   "SELECT * FROM t1"
+  2   "SELECT * FROM t1 WHERE rowid=5"
+  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
+  4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
+}
+
+#-------------------------------------------------------------------------
+# Test the libraries response if some of the entries in the %_node table 
+# are the wrong size.
+#
+create_t1
+populate_t1
+do_test rtreeA-2.1.0 {
+  set nodes [db eval {select nodeno FROM t1_node}]
+  foreach {a b c} $nodes { truncate_node $c 200 }
+} {}
+do_corruption_tests rtreeA-2.1 {
+  1   "SELECT * FROM t1"
+  2   "SELECT * FROM t1 WHERE rowid=5"
+  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
+  4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
+}
+
+create_t1
+populate_t1
+do_test rtreeA-2.2.0 { truncate_node 1 200 } {}
+do_corruption_tests rtreeA-2.2 {
+  1   "SELECT * FROM t1"
+  2   "SELECT * FROM t1 WHERE rowid=5"
+  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
+  4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
+}
+
+#-------------------------------------------------------------------------
+# Set the "depth" of the tree stored on the root node incorrectly. Test
+# that this does not cause any problems.
+#
+create_t1
+populate_t1
+do_test rtreeA-3.1.0.1 { set_tree_depth t1 } {1}
+do_test rtreeA-3.1.0.2 { set_tree_depth t1 3 } {3}
+do_corruption_tests rtreeA-3.1 {
+  1   "SELECT * FROM t1"
+  2   "SELECT * FROM t1 WHERE rowid=5"
+  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
+}
+
+do_test rtreeA-3.2.0 { set_tree_depth t1 1000 } {1000}
+do_corruption_tests rtreeA-3.2 {
+  1   "SELECT * FROM t1"
+  2   "SELECT * FROM t1 WHERE rowid=5"
+  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
+}
+
+create_t1
+populate_t1
+do_test rtreeA-3.3.0 { 
+  execsql { DELETE FROM t1 WHERE rowid = 0 }
+  set_tree_depth t1 65535
+} {65535}
+do_corruption_tests rtreeA-3.3 {
+  1   "SELECT * FROM t1"
+  2   "SELECT * FROM t1 WHERE rowid=5"
+  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
+}
+
+#-------------------------------------------------------------------------
+# Set the "number of entries" field on some nodes incorrectly.
+#
+create_t1
+populate_t1
+do_test rtreeA-4.1.0 { 
+  set_entry_count t1 1 4000
+} {4000}
+do_corruption_tests rtreeA-4.1 {
+  1   "SELECT * FROM t1"
+  2   "SELECT * FROM t1 WHERE rowid=5"
+  3   "INSERT INTO t1 VALUES(1000, 1, 2, 3, 4)"
+  4   "SELECT * FROM t1 WHERE x1<10 AND x2>12"
+}
+
+#-------------------------------------------------------------------------
+# Remove entries from the %_parent table and check that this does not
+# cause a crash.
+#
+create_t1
+populate_t1
+do_execsql_test rtreeA-5.1.0 { DELETE FROM t1_parent } {}
+do_corruption_tests rtreeA-5.1 {
+  1   "DELETE FROM t1 WHERE rowid = 5"
+  2   "DELETE FROM t1"
+}
+
+#-------------------------------------------------------------------------
+# Add some bad entries to the %_parent table.
+#
+create_t1
+populate_t1
+do_execsql_test rtreeA-6.1.0 { 
+  UPDATE t1_parent set parentnode = parentnode+1
+} {}
+do_corruption_tests rtreeA-6.1 {
+  1   "DELETE FROM t1 WHERE rowid = 5"
+  2   "UPDATE t1 SET x1=x1+1, x2=x2+1"
+}
+
+
+finish_test
diff --git a/third_party/sqlite/src/ext/rtree/rtree_perf.tcl b/third_party/sqlite/src/ext/rtree/rtree_perf.tcl
index fa3a4d3..e42e685 100755..100644
--- a/third_party/sqlite/src/ext/rtree/rtree_perf.tcl
+++ b/third_party/sqlite/src/ext/rtree/rtree_perf.tcl
@@ -72,5 +72,3 @@ set rtree_select_time [time {
   }
 }]
 puts "$rtree_select_time"
-
-
diff --git a/third_party/sqlite/src/ext/rtree/rtree_util.tcl b/third_party/sqlite/src/ext/rtree/rtree_util.tcl
index 55482e4..50a1b58 100755..100644
--- a/third_party/sqlite/src/ext/rtree/rtree_util.tcl
+++ b/third_party/sqlite/src/ext/rtree/rtree_util.tcl
@@ -13,8 +13,6 @@
 # analyzing r-tree structures created with this module. It is
 # used by both test procedures and the r-tree viewer application.
 #
-# $Id: rtree_util.tcl,v 1.1 2008/05/26 18:41:54 danielk1977 Exp $
-#
 
 
 #--------------------------------------------------------------------------
@@ -192,4 +190,3 @@ proc rtree_treedump {db zTab} {
   set d [rtree_depth $db $zTab]
   rtree_nodetreedump $db $zTab "" $d 1
 }
-
diff --git a/third_party/sqlite/src/ext/rtree/sqlite3rtree.h b/third_party/sqlite/src/ext/rtree/sqlite3rtree.h
new file mode 100644
index 0000000..cffb300
--- /dev/null
+++ b/third_party/sqlite/src/ext/rtree/sqlite3rtree.h
@@ -0,0 +1,56 @@
+/*
+** 2010 August 30
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#ifndef _SQLITE3RTREE_H_
+#define _SQLITE3RTREE_H_
+
+#include <sqlite3.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry;
+
+/*
+** Register a geometry callback named zGeom that can be used as part of an
+** R-Tree geometry query as follows:
+**
+**   SELECT ... FROM <rtree> WHERE <rtree col> MATCH $zGeom(... params ...)
+*/
+int sqlite3_rtree_geometry_callback(
+  sqlite3 *db,
+  const char *zGeom,
+  int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes),
+  void *pContext
+);
+
+
+/*
+** A pointer to a structure of the following type is passed as the first
+** argument to callbacks registered using rtree_geometry_callback().
+*/
+struct sqlite3_rtree_geometry {
+  void *pContext;                 /* Copy of pContext passed to s_r_g_c() */
+  int nParam;                     /* Size of array aParam[] */
+  double *aParam;                 /* Parameters passed to SQL geom function */
+  void *pUser;                    /* Callback implementation user data */
+  void (*xDelUser)(void *);       /* Called by SQLite to clean up pUser */
+};
+
+
+#ifdef __cplusplus
+}  /* end of the 'extern "C"' block */
+#endif
+
+#endif  /* ifndef _SQLITE3RTREE_H_ */
diff --git a/third_party/sqlite/src/ext/rtree/tkt3363.test b/third_party/sqlite/src/ext/rtree/tkt3363.test
index 11c5192..db05ed5 100644
--- a/third_party/sqlite/src/ext/rtree/tkt3363.test
+++ b/third_party/sqlite/src/ext/rtree/tkt3363.test
@@ -11,11 +11,9 @@
 #
 # The focus of this file is testing that ticket #3363 is fixed.
 #
-# $Id: tkt3363.test,v 1.1 2008/09/08 11:07:03 danielk1977 Exp $
-#
 
 if {![info exists testdir]} {
-  set testdir [file join [file dirname $argv0] .. .. test]
+  set testdir [file join [file dirname [info script]] .. .. test]
 }
 source [file join [file dirname [info script]] rtree_util.tcl]
 source $testdir/tester.tcl
@@ -50,5 +48,3 @@ do_test tkt3363.1.4 {
 } {7}
 
 finish_test
-
-
diff --git a/third_party/sqlite/src/ext/rtree/viewrtree.tcl b/third_party/sqlite/src/ext/rtree/viewrtree.tcl
index 2b4dd1b..794677f 100755..100644
--- a/third_party/sqlite/src/ext/rtree/viewrtree.tcl
+++ b/third_party/sqlite/src/ext/rtree/viewrtree.tcl
@@ -186,4 +186,3 @@ proc cell_report {db zTab iParent iCell} {
 
 view_node
 bind .c <Configure> view_node
-