diff options
| -rw-r--r-- | sql/connection.cc | 56 | ||||
| -rw-r--r-- | sql/connection.h | 22 | ||||
| -rw-r--r-- | third_party/sqlite/README.chromium | 7 | ||||
| -rw-r--r-- | third_party/sqlite/amalgamation/sqlite3.c | 228 | ||||
| -rw-r--r-- | third_party/sqlite/amalgamation/sqlite3.h | 15 | ||||
| -rw-r--r-- | third_party/sqlite/preload-cache.patch | 280 | ||||
| -rw-r--r-- | third_party/sqlite/src/src/build.c | 30 | ||||
| -rw-r--r-- | third_party/sqlite/src/src/pager.c | 172 | ||||
| -rw-r--r-- | third_party/sqlite/src/src/pager.h | 2 | ||||
| -rw-r--r-- | third_party/sqlite/src/src/pcache.c | 2 | ||||
| -rw-r--r-- | third_party/sqlite/src/src/pcache.h | 2 | ||||
| -rw-r--r-- | third_party/sqlite/src/src/sqlite.h.in | 15 |
12 files changed, 65 insertions, 766 deletions
diff --git a/sql/connection.cc b/sql/connection.cc index 0caa9ed..bfea605 100644 --- a/sql/connection.cc +++ b/sql/connection.cc @@ -125,6 +125,22 @@ void InitializeSqlite() { sqlite3_initialize(); } +// Helper to get the sqlite3_file* associated with the "main" database. +int GetSqlite3File(sqlite3* db, sqlite3_file** file) { + *file = NULL; + int rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_FILE_POINTER, file); + if (rc != SQLITE_OK) + return rc; + + // TODO(shess): NULL in file->pMethods has been observed on android_dbg + // content_unittests, even though it should not be possible. + // http://crbug.com/329982 + if (!*file || !(*file)->pMethods) + return SQLITE_ERROR; + + return rc; +} + } // namespace namespace sql { @@ -302,20 +318,33 @@ void Connection::Preload() { return; } - // A statement must be open for the preload command to work. If the meta - // table doesn't exist, it probably means this is a new database and there - // is nothing to preload (so it's OK we do nothing). - if (!DoesTableExist("meta")) + // Use local settings if provided, otherwise use documented defaults. The + // actual results could be fetching via PRAGMA calls. + const int page_size = page_size_ ? page_size_ : 1024; + sqlite3_int64 preload_size = page_size * (cache_size_ ? cache_size_ : 2000); + if (preload_size < 1) return; - Statement dummy(GetUniqueStatement("SELECT * FROM meta")); - if (!dummy.Step()) + + sqlite3_file* file = NULL; + int rc = GetSqlite3File(db_, &file); + if (rc != SQLITE_OK) return; -#if !defined(USE_SYSTEM_SQLITE) - // This function is only defined in Chromium's version of sqlite. - // Do not call it when using system sqlite. - sqlite3_preload(db_); -#endif + sqlite3_int64 file_size = 0; + rc = file->pMethods->xFileSize(file, &file_size); + if (rc != SQLITE_OK) + return; + + // Don't preload more than the file contains. + if (preload_size > file_size) + preload_size = file_size; + + scoped_ptr<char[]> buf(new char[page_size]); + for (sqlite3_int64 pos = 0; pos < file_size; pos += page_size) { + rc = file->pMethods->xRead(file, buf.get(), page_size, pos); + if (rc != SQLITE_OK) + return; + } } void Connection::TrimMemory(bool aggressively) { @@ -429,13 +458,10 @@ bool Connection::Raze() { // the get-go? if (rc == SQLITE_NOTADB || rc == SQLITE_IOERR_SHORT_READ) { sqlite3_file* file = NULL; - rc = sqlite3_file_control(db_, "main", SQLITE_FCNTL_FILE_POINTER, &file); + rc = GetSqlite3File(db_, &file); if (rc != SQLITE_OK) { DLOG(FATAL) << "Failure getting file handle."; return false; - } else if (!file) { - DLOG(FATAL) << "File handle is empty."; - return false; } rc = file->pMethods->xTruncate(file, 0); diff --git a/sql/connection.h b/sql/connection.h index 54c3f8b..5f11e44 100644 --- a/sql/connection.h +++ b/sql/connection.h @@ -187,19 +187,17 @@ class SQL_EXPORT Connection { // an uninitialized or already-closed database. void Close(); - // Pre-loads the first <cache-size> pages into the cache from the file. - // If you expect to soon use a substantial portion of the database, this - // is much more efficient than allowing the pages to be populated organically - // since there is no per-page hard drive seeking. If the file is larger than - // the cache, the last part that doesn't fit in the cache will be brought in - // organically. + // Reads the first <cache-size>*<page-size> bytes of the file to prime the + // filesystem cache. This can be more efficient than faulting pages + // individually. Since this involves blocking I/O, it should only be used if + // the caller will immediately read a substantial amount of data from the + // database. // - // This function assumes your class is using a meta table on the current - // database, as it openes a transaction on the meta table to force the - // database to be initialized. You should feel free to initialize the meta - // table after calling preload since the meta table will already be in the - // database if it exists, and if it doesn't exist, the database won't - // generally exist either. + // TODO(shess): Design a set of histograms or an experiment to inform this + // decision. Preloading should almost always improve later performance + // numbers for this database simply because it pulls operations forward, but + // if the data isn't actually used soon then preloading just slows down + // everything else. void Preload(); // Try to trim the cache memory used by the database. If |aggressively| is diff --git a/third_party/sqlite/README.chromium b/third_party/sqlite/README.chromium index da14b0e..aeacf14 100644 --- a/third_party/sqlite/README.chromium +++ b/third_party/sqlite/README.chromium @@ -59,7 +59,6 @@ applied to a vanilla SQLite (of the version we currently have) to get, in principle, exactly what is checked in: misc.patch -preload-cache.patch safe-tolower.patch fts2.patch fts3.patch @@ -81,7 +80,6 @@ So, e.g. you could do this to apply all our patches to vanilla SQLite: cd sqlite-$LATEST patch -p0 < ../sqlite/misc.patch -patch -p0 < ../sqlite/preload-cache.patch patch -p0 < ../sqlite/fts2.patch patch -p0 < ../sqlite/fts3.patch patch -p0 < ../sqlite/fts3_85522.patch @@ -167,11 +165,6 @@ Changes from Chrome: our ICU data. shell.c has been modifed to call into these files. - fts2_icu.c and fts3_icu.c have a critical bug. U8_NEXT is used over a UTF-16 string. It's rep$ by U16_NEXT (jungshik) - - Added a new function sqlite3_preload we use to prime the database cache. It - allows much faster performance by reading the file in one contiguous - operation rather than bringing it in organically, which involves a lot of - seeking. This change also required sqlite3PcacheGetCachesize to be compiled - even outside SQLITE_TEST. - Added a new function chromium_sqlite3_initialize_win_sqlite3_file() at the end of os_win.c. It allows the Windows-specific Chromium VFS to reuse most of the win32 SQLite VFS. diff --git a/third_party/sqlite/amalgamation/sqlite3.c b/third_party/sqlite/amalgamation/sqlite3.c index 91e98680..608cb26 100644 --- a/third_party/sqlite/amalgamation/sqlite3.c +++ b/third_party/sqlite/amalgamation/sqlite3.c @@ -5234,21 +5234,6 @@ struct sqlite3_index_info { double estimatedCost; /* Estimated cost of using this index */ }; -/* Begin preload-cache.patch for Chromium */ -/* -** Preload the databases into the pager cache, up to the maximum size of the -** pager cache. -** -** For a database to be loaded successfully, the pager must be active. That is, -** there must be an open statement on that database. See sqlite3pager_loadall -** -** There might be many databases attached to the given connection. We iterate -** them all and try to load them. If none are loadable successfully, we return -** an error. Otherwise, we return OK. -*/ -SQLITE_API int sqlite3_preload(sqlite3 *db); -/* End preload-cache.patch for Chromium */ - /* ** CAPI3REF: Virtual Table Constraint Operator Codes ** @@ -8459,8 +8444,6 @@ SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager*); SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*); SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager*); SQLITE_PRIVATE int sqlite3PagerNosync(Pager*); -/* This function is for preload-cache.patch for Chromium: */ -SQLITE_PRIVATE int sqlite3PagerLoadall(Pager*); SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); @@ -8631,7 +8614,9 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** of the suggested cache-sizes. */ SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *, int); +#ifdef SQLITE_TEST SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *); +#endif #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* Try to return memory used by the pcache module to the main memory heap */ @@ -34893,12 +34878,14 @@ SQLITE_PRIVATE int sqlite3PcachePagecount(PCache *pCache){ return nPage; } +#ifdef SQLITE_TEST /* ** Get the suggested cache-size value. */ SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *pCache){ return pCache->nMax; } +#endif /* ** Set the suggested cache-size value. @@ -37234,16 +37221,6 @@ static int pagerUseWal(Pager *pPager){ # define pagerBeginReadTransaction(z) SQLITE_OK #endif -/* Begin preload-cache.patch for Chromium */ -/* See comments above the definition. */ -SQLITE_PRIVATE int sqlite3PagerAcquire2( - Pager *pPager, - Pgno pgno, - DbPage **ppPage, - int noContent, - unsigned char *pDataToFill); -/* End preload-cache.patch for Chromium */ - #ifndef NDEBUG /* ** Usage: @@ -41413,25 +41390,6 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( DbPage **ppPage, /* Write a pointer to the page here */ int noContent /* Do not bother reading content from disk if true */ ){ - /* This just passes through to our modified version with NULL data. */ - return sqlite3PagerAcquire2(pPager, pgno, ppPage, noContent, 0); -} - -/* -** This is an internal version of sqlite3PagerAcquire that takes an extra -** parameter of data to use to fill the page with. This allows more efficient -** filling for preloaded data. If this extra parameter is NULL, we'll go to -** the file. -** -** See sqlite3PagerLoadall which uses this function. -*/ -SQLITE_PRIVATE int sqlite3PagerAcquire2( - Pager *pPager, /* The pager open on the database file */ - Pgno pgno, /* Page number to fetch */ - DbPage **ppPage, /* Write a pointer to the page here */ - int noContent, /* Do not bother reading content from disk if true */ - unsigned char* pDataToFill -){ int rc; PgHdr *pPg; @@ -41507,17 +41465,9 @@ SQLITE_PRIVATE int sqlite3PagerAcquire2( IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ assert( pPg->pPager==pPager ); - if( pDataToFill ){ - /* Just copy from the given memory */ - memcpy(pPg->pData, pDataToFill, pPager->pageSize); - CODEC1(pPager, pPg->pData, pPg->pgno, 3, rc = SQLITE_NOMEM; - goto pager_acquire_err); - }else{ - /* Load from disk (old regular sqlite code path) */ - rc = readDbPage(pPg); - if( rc!=SQLITE_OK ){ - goto pager_acquire_err; - } + rc = readDbPage(pPg); + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; } } pager_set_pagehash(pPg); @@ -42939,135 +42889,6 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i } #endif -/* Begin preload-cache.patch for Chromium */ -#if 1 -/* NOTE(shess): Testing to see if simply reading the data into the - * filesystem buffers will have the positive speed impact without the - * negative memory impact. - */ -SQLITE_PRIVATE int sqlite3PagerLoadall(Pager* pPager) -{ - int i, pageSize, loadPages, rc; - unsigned char *fileData; - - /* TODO(shess): This test may not be relevant for this - * implementation, but keep the invariant consistent. - */ - pageSize = pPager->pageSize; - if (pPager->dbSize < 0 || pageSize < 0) { - /* pager not initialized, this means a statement is not open */ - return SQLITE_MISUSE; - } - - /* Allocate a buffer to read pages into. */ - /* TODO(shess): No need to read by page, this could be a fixed-size - * buffer on stack. - */ - fileData = sqlite3Malloc(pageSize); - if (!fileData) - return SQLITE_NOMEM; - - /* Load the smaller of the entire cache or the entire database. */ - loadPages = sqlite3PcacheGetCachesize(pPager->pPCache); - if (loadPages > pPager->dbSize) - loadPages = pPager->dbSize; - - /* Read database page by page. */ - rc = SQLITE_OK; - for(i=0; i < loadPages; i++) { - rc = sqlite3OsRead(pPager->fd, fileData, pageSize, i*pageSize); - if (rc != SQLITE_OK) - break; - } - sqlite3_free(fileData); - return rc; -} -#else -/** -** When making large allocations, there is no need to stress the heap and -** potentially hold its lock while we allocate a bunch of memory. If we know -** the allocation will be large, go directly to the OS instead of the heap. -**/ -static void* allocLarge(size_t size) { -#if SQLITE_OS_WIN - return VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); -#else - return sqlite3Malloc(size); -#endif -} - -static void freeLarge(void* ptr) { -#if SQLITE_OS_WIN - VirtualFree(ptr, 0, MEM_RELEASE); -#else - sqlite3_free(ptr); -#endif -} - -/** -** Addition: This will attempt to populate the database cache with -** the first N bytes of the file, where N is the total size of the cache. -** Because we can load this as one chunk from the disk, this is much faster -** than loading a subset of the pages one at a time in random order. -** -** The pager must be initialized before this function is called. This means a -* statement must be open that has initialized the pager and is keeping the -** cache in memory. -**/ -SQLITE_PRIVATE int sqlite3PagerLoadall(Pager* pPager) -{ - int i; - int rc; - int nMax; - int loadSize; - int loadPages; - unsigned char *fileData; - - if (pPager->dbSize < 0 || pPager->pageSize < 0) { - /* pager not initialized, this means a statement is not open */ - return SQLITE_MISUSE; - } - - /* compute sizes */ - nMax = sqlite3PcacheGetCachesize(pPager->pPCache); - if (nMax < pPager->dbSize) - loadPages = nMax; - else - loadPages = pPager->dbSize; - loadSize = loadPages * pPager->pageSize; - - /* load the file as one chunk */ - fileData = allocLarge(loadSize); - if (! fileData) - return SQLITE_NOMEM; - rc = sqlite3OsRead(pPager->fd, fileData, loadSize, 0); - if (rc != SQLITE_OK) { - freeLarge(fileData); - return rc; - } - - /* Copy the data to each page. Note that the page numbers we pass to _get - * are one-based, 0 is a marker for no page. We also need to check that we - * haven't loaded more pages than the cache can hold total. There may have - * already been a few pages loaded before, so we may fill the cache before - * loading all of the pages we want to. - */ - for(i=1; - i <= loadPages && sqlite3PcachePagecount(pPager->pPCache) < nMax; - i++) { - DbPage *pPage = 0; - rc = sqlite3PagerAcquire2(pPager, i, &pPage, 0, - &fileData[(i-1)*(i64)pPager->pageSize]); - if (rc != SQLITE_OK) - break; - sqlite3PagerUnref(pPage); - } - freeLarge(fileData); - return SQLITE_OK; -} -#endif -/* End preload-cache.patch for Chromium */ - /* ** Return a pointer to the data for the specified page. */ @@ -77042,7 +76863,6 @@ SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ ** ROLLBACK */ - /* ** This routine is called when a new SQL statement is beginning to ** be parsed. Initialize the pParse structure as needed. @@ -80775,33 +80595,6 @@ SQLITE_PRIVATE KeyInfo *sqlite3IndexKeyinfo(Parse *pParse, Index *pIdx){ return pKey; } -/* Begin preload-cache.patch for Chromium */ -/* See declaration in sqlite3.h for information */ -SQLITE_API int sqlite3_preload(sqlite3 *db) -{ - Pager *pPager; - Btree *pBt; - int rc; - int i; - int dbsLoaded = 0; - - for(i=0; i<db->nDb; i++) { - pBt = db->aDb[i].pBt; - if( !pBt ) - continue; - pPager = sqlite3BtreePager(pBt); - if( pPager ) { - rc = sqlite3PagerLoadall(pPager); - if (rc == SQLITE_OK) - dbsLoaded++; - } - } - if (dbsLoaded == 0) - return SQLITE_ERROR; - return SQLITE_OK; -} -/* End preload-cache.patch for Chromium */ - /************** End of build.c ***********************************************/ /************** Begin file callback.c ****************************************/ /* @@ -112179,13 +111972,6 @@ static int recoverFilter( FNENTRY(); - // TODO(shess): http://crbug.com/326646 implies that pCursor->pLeafCursor is - // NULL here. It shouldn't be possible, until it is figured out prevent the - // crash. - if( !pCursor->pLeafCursor ){ - return SQLITE_ERROR; - } - /* Load the first cell, and iterate forward if it's not valid. */ /* TODO(shess): What happens if no cells at all are valid? */ rc = leafCursorCellDecode(pCursor->pLeafCursor); diff --git a/third_party/sqlite/amalgamation/sqlite3.h b/third_party/sqlite/amalgamation/sqlite3.h index 5d56618..91ac914 100644 --- a/third_party/sqlite/amalgamation/sqlite3.h +++ b/third_party/sqlite/amalgamation/sqlite3.h @@ -4691,21 +4691,6 @@ struct sqlite3_index_info { double estimatedCost; /* Estimated cost of using this index */ }; -/* Begin preload-cache.patch for Chromium */ -/* -** Preload the databases into the pager cache, up to the maximum size of the -** pager cache. -** -** For a database to be loaded successfully, the pager must be active. That is, -** there must be an open statement on that database. See sqlite3pager_loadall -** -** There might be many databases attached to the given connection. We iterate -** them all and try to load them. If none are loadable successfully, we return -** an error. Otherwise, we return OK. -*/ -SQLITE_API int sqlite3_preload(sqlite3 *db); -/* End preload-cache.patch for Chromium */ - /* ** CAPI3REF: Virtual Table Constraint Operator Codes ** diff --git a/third_party/sqlite/preload-cache.patch b/third_party/sqlite/preload-cache.patch deleted file mode 100644 index 7a361d1..0000000 --- a/third_party/sqlite/preload-cache.patch +++ /dev/null @@ -1,280 +0,0 @@ -Adds a new API function sqlite3_preload(). This fills the page cache -with the first pages of the database. - -Index: src/build.c -=================================================================== ---- src/build.c 2009-09-11 07:02:46.000000000 -0700 -+++ src/build.c 2009-09-14 18:16:46.000000000 -0700 -@@ -26,6 +26,9 @@ - */ - #include "sqliteInt.h" - -+#include "pager.h" -+#include "btree.h" -+ - /* - ** This routine is called when a new SQL statement is beginning to - ** be parsed. Initialize the pParse structure as needed. -@@ -3659,3 +3662,30 @@ - } - return pKey; - } -+ -+/* Begin preload-cache.patch for Chromium */ -+/* See declaration in sqlite3.h for information */ -+int sqlite3_preload(sqlite3 *db) -+{ -+ Pager *pPager; -+ Btree *pBt; -+ int rc; -+ int i; -+ int dbsLoaded = 0; -+ -+ for(i=0; i<db->nDb; i++) { -+ pBt = db->aDb[i].pBt; -+ if( !pBt ) -+ continue; -+ pPager = sqlite3BtreePager(pBt); -+ if( pPager ) { -+ rc = sqlite3PagerLoadall(pPager); -+ if (rc == SQLITE_OK) -+ dbsLoaded++; -+ } -+ } -+ if (dbsLoaded == 0) -+ return SQLITE_ERROR; -+ return SQLITE_OK; -+} -+/* End preload-cache.patch for Chromium */ -Index: src/sqlite3.h.in -=================================================================== ---- src/sqlite.h.in 2009-09-09 07:03:20.000000000 -0700 -+++ src/sqlite.h.in 2009-09-15 11:34:26.000000000 -0700 -@@ -4677,6 +4677,21 @@ - */ - int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); - -+/* Begin preload-cache.patch for Chromium */ -+/* -+** Preload the databases into the pager cache, up to the maximum size of the -+** pager cache. -+** -+** For a database to be loaded successfully, the pager must be active. That is, -+** there must be an open statement on that database. See sqlite3pager_loadall -+** -+** There might be many databases attached to the given connection. We iterate -+** them all and try to load them. If none are loadable successfully, we return -+** an error. Otherwise, we return OK. -+*/ -+int sqlite3_preload(sqlite3 *db); -+/* End preload-cache.patch for Chromium */ -+ - /* - ** CAPI3REF: Virtual File System Objects {H11200} <S20100> - ** -Index: src/pager.c -=================================================================== ---- src/pager.c 2009-09-07 08:58:09.000000000 -0700 -+++ src/pager.c 2009-09-15 16:43:07.000000000 -0700 -@@ -388,6 +388,16 @@ - */ - #define PAGER_MAX_PGNO 2147483647 - -+/* Begin preload-cache.patch for Chromium */ -+/* See comments above the definition. */ -+int sqlite3PagerAcquire2( -+ Pager *pPager, -+ Pgno pgno, -+ DbPage **ppPage, -+ int noContent, -+ unsigned char *pDataToFill); -+/* End preload-cache.patch for Chromium */ -+ - #ifndef NDEBUG - /* - ** Usage: -@@ -3788,6 +3798,25 @@ - DbPage **ppPage, /* Write a pointer to the page here */ - int noContent /* Do not bother reading content from disk if true */ - ){ -+ /* This just passes through to our modified version with NULL data. */ -+ return sqlite3PagerAcquire2(pPager, pgno, ppPage, noContent, 0); -+} -+ -+/* -+** This is an internal version of sqlite3PagerAcquire that takes an extra -+** parameter of data to use to fill the page with. This allows more efficient -+** filling for preloaded data. If this extra parameter is NULL, we'll go to -+** the file. -+** -+** See sqlite3PagerLoadall which uses this function. -+*/ -+int sqlite3PagerAcquire2( -+ Pager *pPager, /* The pager open on the database file */ -+ Pgno pgno, /* Page number to fetch */ -+ DbPage **ppPage, /* Write a pointer to the page here */ -+ int noContent, /* Do not bother reading content from disk if true */ -+ unsigned char* pDataToFill -+){ - int rc; - PgHdr *pPg; - -@@ -3870,9 +3899,17 @@ - IOTRACE(("ZERO %p %d\n", pPager, pgno)); - }else{ - assert( pPg->pPager==pPager ); -- rc = readDbPage(pPg); -- if( rc!=SQLITE_OK ){ -- goto pager_acquire_err; -+ if( pDataToFill ){ -+ /* Just copy from the given memory */ -+ memcpy(pPg->pData, pDataToFill, pPager->pageSize); -+ CODEC1(pPager, pPg->pData, pPg->pgno, 3, rc = SQLITE_NOMEM; -+ goto pager_acquire_err); -+ }else{ -+ /* Load from disk (old regular sqlite code path) */ -+ rc = readDbPage(pPg); -+ if( rc!=SQLITE_OK ){ -+ goto pager_acquire_err; -+ } - } - } - #ifdef SQLITE_CHECK_PAGES -@@ -5221,6 +5258,91 @@ - } - #endif - -+/* Begin preload-cache.patch for Chromium */ -+/** -+** When making large allocations, there is no need to stress the heap and -+** potentially hold its lock while we allocate a bunch of memory. If we know -+** the allocation will be large, go directly to the OS instead of the heap. -+**/ -+static void* allocLarge(size_t size) { -+#if SQLITE_OS_WIN -+ return VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); -+#else -+ return sqlite3Malloc(size); -+#endif -+} -+ -+static void freeLarge(void* ptr) { -+#if SQLITE_OS_WIN -+ VirtualFree(ptr, 0, MEM_RELEASE); -+#else -+ sqlite3_free(ptr); -+#endif -+} -+ -+/** -+** Addition: This will attempt to populate the database cache with -+** the first N bytes of the file, where N is the total size of the cache. -+** Because we can load this as one chunk from the disk, this is much faster -+** than loading a subset of the pages one at a time in random order. -+** -+** The pager must be initialized before this function is called. This means a -+* statement must be open that has initialized the pager and is keeping the -+** cache in memory. -+**/ -+int sqlite3PagerLoadall(Pager* pPager) -+{ -+ int i; -+ int rc; -+ int nMax; -+ int loadSize; -+ int loadPages; -+ unsigned char *fileData; -+ -+ if (pPager->dbSize < 0 || pPager->pageSize < 0) { -+ /* pager not initialized, this means a statement is not open */ -+ return SQLITE_MISUSE; -+ } -+ -+ /* compute sizes */ -+ nMax = sqlite3PcacheGetCachesize(pPager->pPCache); -+ if (nMax < pPager->dbSize) -+ loadPages = nMax; -+ else -+ loadPages = pPager->dbSize; -+ loadSize = loadPages * pPager->pageSize; -+ -+ /* load the file as one chunk */ -+ fileData = allocLarge(loadSize); -+ if (! fileData) -+ return SQLITE_NOMEM; -+ rc = sqlite3OsRead(pPager->fd, fileData, loadSize, 0); -+ if (rc != SQLITE_OK) { -+ freeLarge(fileData); -+ return rc; -+ } -+ -+ /* Copy the data to each page. Note that the page numbers we pass to _get -+ * are one-based, 0 is a marker for no page. We also need to check that we -+ * haven't loaded more pages than the cache can hold total. There may have -+ * already been a few pages loaded before, so we may fill the cache before -+ * loading all of the pages we want to. -+ */ -+ for(i=1; -+ i <= loadPages && sqlite3PcachePagecount(pPager->pPCache) < nMax; -+ i++) { -+ DbPage *pPage = 0; -+ rc = sqlite3PagerAcquire2(pPager, i, &pPage, 0, -+ &fileData[(i-1)*(i64)pPager->pageSize]); -+ if (rc != SQLITE_OK) -+ break; -+ sqlite3PagerUnref(pPage); -+ } -+ freeLarge(fileData); -+ return SQLITE_OK; -+} -+/* End preload-cache.patch for Chromium */ -+ - /* - ** Return a pointer to the data for the specified page. - */ -Index: src/pager.h -=================================================================== ---- src/pager.h 2009-09-04 13:37:42.000000000 -0700 -+++ src/pager.h 2009-09-15 11:31:55.000000000 -0700 -@@ -143,6 +143,8 @@ - sqlite3_file *sqlite3PagerFile(Pager*); - const char *sqlite3PagerJournalname(Pager*); - int sqlite3PagerNosync(Pager*); -+/* This function is for preload-cache.patch for Chromium: */ -+int sqlite3PagerLoadall(Pager*); - void *sqlite3PagerTempSpace(Pager*); - int sqlite3PagerIsMemdb(Pager*); - -Index: src/pcache.c -=================================================================== ---- src/pcache.c 2009-09-04 13:37:42.000000000 -0700 -+++ src/pcache.c 2009-09-15 16:41:55.000000000 -0700 -@@ -542,14 +542,12 @@ - return nPage; - } - --#ifdef SQLITE_TEST - /* - ** Get the suggested cache-size value. - */ - int sqlite3PcacheGetCachesize(PCache *pCache){ - return pCache->nMax; - } --#endif - - /* - ** Set the suggested cache-size value. -Index: src/pcache.h -=================================================================== ---- src/pcache.h 2009-09-04 13:37:42.000000000 -0700 -+++ src/pcache.h 2009-09-15 16:41:52.000000000 -0700 -@@ -139,9 +139,7 @@ - ** of the suggested cache-sizes. - */ - void sqlite3PcacheSetCachesize(PCache *, int); --#ifdef SQLITE_TEST - int sqlite3PcacheGetCachesize(PCache *); --#endif - - #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - /* Try to return memory used by the pcache module to the main memory heap */ diff --git a/third_party/sqlite/src/src/build.c b/third_party/sqlite/src/src/build.c index 4f4f8ed..25a74ca 100644 --- a/third_party/sqlite/src/src/build.c +++ b/third_party/sqlite/src/src/build.c @@ -24,9 +24,6 @@ */ #include "sqliteInt.h" -#include "pager.h" -#include "btree.h" - /* ** This routine is called when a new SQL statement is beginning to ** be parsed. Initialize the pParse structure as needed. @@ -3758,30 +3755,3 @@ KeyInfo *sqlite3IndexKeyinfo(Parse *pParse, Index *pIdx){ } return pKey; } - -/* Begin preload-cache.patch for Chromium */ -/* See declaration in sqlite3.h for information */ -int sqlite3_preload(sqlite3 *db) -{ - Pager *pPager; - Btree *pBt; - int rc; - int i; - int dbsLoaded = 0; - - for(i=0; i<db->nDb; i++) { - pBt = db->aDb[i].pBt; - if( !pBt ) - continue; - pPager = sqlite3BtreePager(pBt); - if( pPager ) { - rc = sqlite3PagerLoadall(pPager); - if (rc == SQLITE_OK) - dbsLoaded++; - } - } - if (dbsLoaded == 0) - return SQLITE_ERROR; - return SQLITE_OK; -} -/* End preload-cache.patch for Chromium */ diff --git a/third_party/sqlite/src/src/pager.c b/third_party/sqlite/src/src/pager.c index a418966..a4fe318 100644 --- a/third_party/sqlite/src/src/pager.c +++ b/third_party/sqlite/src/src/pager.c @@ -791,16 +791,6 @@ static int pagerUseWal(Pager *pPager){ # define pagerBeginReadTransaction(z) SQLITE_OK #endif -/* Begin preload-cache.patch for Chromium */ -/* See comments above the definition. */ -int sqlite3PagerAcquire2( - Pager *pPager, - Pgno pgno, - DbPage **ppPage, - int noContent, - unsigned char *pDataToFill); -/* End preload-cache.patch for Chromium */ - #ifndef NDEBUG /* ** Usage: @@ -4970,25 +4960,6 @@ int sqlite3PagerAcquire( DbPage **ppPage, /* Write a pointer to the page here */ int noContent /* Do not bother reading content from disk if true */ ){ - /* This just passes through to our modified version with NULL data. */ - return sqlite3PagerAcquire2(pPager, pgno, ppPage, noContent, 0); -} - -/* -** This is an internal version of sqlite3PagerAcquire that takes an extra -** parameter of data to use to fill the page with. This allows more efficient -** filling for preloaded data. If this extra parameter is NULL, we'll go to -** the file. -** -** See sqlite3PagerLoadall which uses this function. -*/ -int sqlite3PagerAcquire2( - Pager *pPager, /* The pager open on the database file */ - Pgno pgno, /* Page number to fetch */ - DbPage **ppPage, /* Write a pointer to the page here */ - int noContent, /* Do not bother reading content from disk if true */ - unsigned char* pDataToFill -){ int rc; PgHdr *pPg; @@ -5064,17 +5035,9 @@ int sqlite3PagerAcquire2( IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ assert( pPg->pPager==pPager ); - if( pDataToFill ){ - /* Just copy from the given memory */ - memcpy(pPg->pData, pDataToFill, pPager->pageSize); - CODEC1(pPager, pPg->pData, pPg->pgno, 3, rc = SQLITE_NOMEM; - goto pager_acquire_err); - }else{ - /* Load from disk (old regular sqlite code path) */ - rc = readDbPage(pPg); - if( rc!=SQLITE_OK ){ - goto pager_acquire_err; - } + rc = readDbPage(pPg); + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; } } pager_set_pagehash(pPg); @@ -6496,135 +6459,6 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ } #endif -/* Begin preload-cache.patch for Chromium */ -#if 1 -/* NOTE(shess): Testing to see if simply reading the data into the - * filesystem buffers will have the positive speed impact without the - * negative memory impact. - */ -int sqlite3PagerLoadall(Pager* pPager) -{ - int i, pageSize, loadPages, rc; - unsigned char *fileData; - - /* TODO(shess): This test may not be relevant for this - * implementation, but keep the invariant consistent. - */ - pageSize = pPager->pageSize; - if (pPager->dbSize < 0 || pageSize < 0) { - /* pager not initialized, this means a statement is not open */ - return SQLITE_MISUSE; - } - - /* Allocate a buffer to read pages into. */ - /* TODO(shess): No need to read by page, this could be a fixed-size - * buffer on stack. - */ - fileData = sqlite3Malloc(pageSize); - if (!fileData) - return SQLITE_NOMEM; - - /* Load the smaller of the entire cache or the entire database. */ - loadPages = sqlite3PcacheGetCachesize(pPager->pPCache); - if (loadPages > pPager->dbSize) - loadPages = pPager->dbSize; - - /* Read database page by page. */ - rc = SQLITE_OK; - for(i=0; i < loadPages; i++) { - rc = sqlite3OsRead(pPager->fd, fileData, pageSize, i*pageSize); - if (rc != SQLITE_OK) - break; - } - sqlite3_free(fileData); - return rc; -} -#else -/** -** When making large allocations, there is no need to stress the heap and -** potentially hold its lock while we allocate a bunch of memory. If we know -** the allocation will be large, go directly to the OS instead of the heap. -**/ -static void* allocLarge(size_t size) { -#if SQLITE_OS_WIN - return VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); -#else - return sqlite3Malloc(size); -#endif -} - -static void freeLarge(void* ptr) { -#if SQLITE_OS_WIN - VirtualFree(ptr, 0, MEM_RELEASE); -#else - sqlite3_free(ptr); -#endif -} - -/** -** Addition: This will attempt to populate the database cache with -** the first N bytes of the file, where N is the total size of the cache. -** Because we can load this as one chunk from the disk, this is much faster -** than loading a subset of the pages one at a time in random order. -** -** The pager must be initialized before this function is called. This means a -* statement must be open that has initialized the pager and is keeping the -** cache in memory. -**/ -int sqlite3PagerLoadall(Pager* pPager) -{ - int i; - int rc; - int nMax; - int loadSize; - int loadPages; - unsigned char *fileData; - - if (pPager->dbSize < 0 || pPager->pageSize < 0) { - /* pager not initialized, this means a statement is not open */ - return SQLITE_MISUSE; - } - - /* compute sizes */ - nMax = sqlite3PcacheGetCachesize(pPager->pPCache); - if (nMax < pPager->dbSize) - loadPages = nMax; - else - loadPages = pPager->dbSize; - loadSize = loadPages * pPager->pageSize; - - /* load the file as one chunk */ - fileData = allocLarge(loadSize); - if (! fileData) - return SQLITE_NOMEM; - rc = sqlite3OsRead(pPager->fd, fileData, loadSize, 0); - if (rc != SQLITE_OK) { - freeLarge(fileData); - return rc; - } - - /* Copy the data to each page. Note that the page numbers we pass to _get - * are one-based, 0 is a marker for no page. We also need to check that we - * haven't loaded more pages than the cache can hold total. There may have - * already been a few pages loaded before, so we may fill the cache before - * loading all of the pages we want to. - */ - for(i=1; - i <= loadPages && sqlite3PcachePagecount(pPager->pPCache) < nMax; - i++) { - DbPage *pPage = 0; - rc = sqlite3PagerAcquire2(pPager, i, &pPage, 0, - &fileData[(i-1)*(i64)pPager->pageSize]); - if (rc != SQLITE_OK) - break; - sqlite3PagerUnref(pPage); - } - freeLarge(fileData); - return SQLITE_OK; -} -#endif -/* End preload-cache.patch for Chromium */ - /* ** Return a pointer to the data for the specified page. */ diff --git a/third_party/sqlite/src/src/pager.h b/third_party/sqlite/src/src/pager.h index 211fe8c..eab7dda 100644 --- a/third_party/sqlite/src/src/pager.h +++ b/third_party/sqlite/src/src/pager.h @@ -153,8 +153,6 @@ const sqlite3_vfs *sqlite3PagerVfs(Pager*); sqlite3_file *sqlite3PagerFile(Pager*); const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); -/* This function is for preload-cache.patch for Chromium: */ -int sqlite3PagerLoadall(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); diff --git a/third_party/sqlite/src/src/pcache.c b/third_party/sqlite/src/src/pcache.c index 6dec550..242f307 100644 --- a/third_party/sqlite/src/src/pcache.c +++ b/third_party/sqlite/src/src/pcache.c @@ -553,12 +553,14 @@ int sqlite3PcachePagecount(PCache *pCache){ return nPage; } +#ifdef SQLITE_TEST /* ** Get the suggested cache-size value. */ int sqlite3PcacheGetCachesize(PCache *pCache){ return pCache->nMax; } +#endif /* ** Set the suggested cache-size value. diff --git a/third_party/sqlite/src/src/pcache.h b/third_party/sqlite/src/src/pcache.h index a9bea98..33735d2 100644 --- a/third_party/sqlite/src/src/pcache.h +++ b/third_party/sqlite/src/src/pcache.h @@ -137,7 +137,9 @@ void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)); ** of the suggested cache-sizes. */ void sqlite3PcacheSetCachesize(PCache *, int); +#ifdef SQLITE_TEST int sqlite3PcacheGetCachesize(PCache *); +#endif #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* Try to return memory used by the pcache module to the main memory heap */ diff --git a/third_party/sqlite/src/src/sqlite.h.in b/third_party/sqlite/src/src/sqlite.h.in index fb76659..00c8510 100644 --- a/third_party/sqlite/src/src/sqlite.h.in +++ b/third_party/sqlite/src/src/sqlite.h.in @@ -4686,21 +4686,6 @@ struct sqlite3_index_info { double estimatedCost; /* Estimated cost of using this index */ }; -/* Begin preload-cache.patch for Chromium */ -/* -** Preload the databases into the pager cache, up to the maximum size of the -** pager cache. -** -** For a database to be loaded successfully, the pager must be active. That is, -** there must be an open statement on that database. See sqlite3pager_loadall -** -** There might be many databases attached to the given connection. We iterate -** them all and try to load them. If none are loadable successfully, we return -** an error. Otherwise, we return OK. -*/ -int sqlite3_preload(sqlite3 *db); -/* End preload-cache.patch for Chromium */ - /* ** CAPI3REF: Virtual Table Constraint Operator Codes ** |