[sql] Consider fresh databases suitable for memory-mapped I/O.

GetAppropriateMmapSize() is called to validate the database as
error-free before enabling memory-mapped mode.  The validation status is
stored in the [meta] table, since new databases have no [meta] table it
previously defaulted to not turning on memory-mapping.  Then on second
open the file would be validated.

This change uses the lack of [meta] table as a signal that the file is
new and should be safe to memory-map.  Additionally the code creating
[meta] pre-populates that setting.

Additionally disable memory-mapped mode for databases which do not make
use of the [meta] table.  This makes the existing settings explicit.

BUG=537742, 555578
TBR=rdevlin.cronin@chromium.org, michaeln@chromium.org, zea@chromium.org, sky@chromium.org

Review URL: https://codereview.chromium.org/1533703002

Cr-Commit-Position: refs/heads/master@{#365959}

5 files changed, 171 insertions, 47 deletions

diff --git a/sql/connection.cc b/sql/connection.cc
index 825d319..1213a77f 100644
--- a/sql/connection.cc
+++ b/sql/connection.cc
@@ -24,6 +24,7 @@
 #include "base/synchronization/lock.h"
 #include "base/trace_event/memory_dump_manager.h"
 #include "base/trace_event/process_memory_dump.h"
+#include "sql/meta_table.h"
 #include "sql/statement.h"
 #include "third_party/sqlite/sqlite3.h"
 
@@ -458,6 +459,7 @@ bool Connection::Open(const base::FilePath& path) {
               base::HistogramBase::kUmaTargetedHistogramFlag);
       if (histogram)
         histogram->Add(sample);
+      UMA_HISTOGRAM_COUNTS("Sqlite.SizeKB", sample);
     }
   }
 
@@ -883,54 +885,43 @@ std::string Connection::CollectCorruptionInfo() {
 size_t Connection::GetAppropriateMmapSize() {
   AssertIOAllowed();
 
-  // TODO(shess): Using sql::MetaTable seems indicated, but mixing
-  // sql::MetaTable and direct access seems error-prone.  It might make sense to
-  // simply integrate sql::MetaTable functionality into sql::Connection.
-
 #if defined(OS_IOS)
   // iOS SQLite does not support memory mapping.
   return 0;
 #endif
 
-  // If the database doesn't have a place to track progress, assume the worst.
-  // This will happen when new databases are created.
-  if (!DoesTableExist("meta")) {
+  // How much to map if no errors are found.  50MB encompasses the 99th
+  // percentile of Chrome databases in the wild, so this should be good.
+  const size_t kMmapEverything = 256 * 1024 * 1024;
+
+  // If the database doesn't have a place to track progress, assume the best.
+  // This will happen when new databases are created, or if a database doesn't
+  // use a meta table.  sql::MetaTable::Init() will preload kMmapSuccess.
+  // TODO(shess): Databases not using meta include:
+  //   DOMStorageDatabase (localstorage)
+  //   ActivityDatabase (extensions activity log)
+  //   PredictorDatabase (prefetch and autocomplete predictor data)
+  //   SyncDirectory (sync metadata storage)
+  // For now, these all have mmap disabled to allow other databases to get the
+  // default-enable path.  sqlite-diag could be an alternative for all but
+  // DOMStorageDatabase, which creates many small databases.
+  // http://crbug.com/537742
+  if (!MetaTable::DoesTableExist(this)) {
     RecordOneEvent(EVENT_MMAP_META_MISSING);
-    return 0;
+    return kMmapEverything;
   }
 
-  // Key into meta table to get status from a previous run.  The value
-  // represents how much data in bytes has successfully been read from the
-  // database.  |kMmapFailure| indicates that there was a read error and the
-  // database should not be memory-mapped, while |kMmapSuccess| indicates that
-  // the entire file was read at some point and can be memory-mapped without
-  // constraint.
-  const char* kMmapStatusKey = "mmap_status";
-  static const sqlite3_int64 kMmapFailure = -2;
-  static const sqlite3_int64 kMmapSuccess = -1;
-
-  // Start reading from 0 unless status is found in meta table.
-  sqlite3_int64 mmap_ofs = 0;
-
-  // Retrieve the current status.  It is fine for the status to be missing
-  // entirely, but any error prevents memory-mapping.
-  {
-    const char* kMmapStatusSql = "SELECT value FROM meta WHERE key = ?";
-    Statement s(GetUniqueStatement(kMmapStatusSql));
-    s.BindString(0, kMmapStatusKey);
-    if (s.Step()) {
-      mmap_ofs = s.ColumnInt64(0);
-    } else if (!s.Succeeded()) {
-      RecordOneEvent(EVENT_MMAP_META_FAILURE_READ);
-      return 0;
-    }
+  int64_t mmap_ofs = 0;
+  if (!MetaTable::GetMmapStatus(this, &mmap_ofs)) {
+    RecordOneEvent(EVENT_MMAP_META_FAILURE_READ);
+    return 0;
   }
 
   // Database read failed in the past, don't memory map.
-  if (mmap_ofs == kMmapFailure) {
+  if (mmap_ofs == MetaTable::kMmapFailure) {
     RecordOneEvent(EVENT_MMAP_FAILED);
     return 0;
-  } else if (mmap_ofs != kMmapSuccess) {
+  } else if (mmap_ofs != MetaTable::kMmapSuccess) {
     // Continue reading from previous offset.
     DCHECK_GE(mmap_ofs, 0);
 
@@ -981,7 +972,7 @@ size_t Connection::GetAppropriateMmapSize() {
           break;
         } else {
           // TODO(shess): Consider calling OnSqliteError().
-          mmap_ofs = kMmapFailure;
+          mmap_ofs = MetaTable::kMmapFailure;
           break;
         }
       }
@@ -989,20 +980,16 @@ size_t Connection::GetAppropriateMmapSize() {
       // Log these events after update to distinguish meta update failure.
       Events event;
       if (mmap_ofs >= db_size) {
-        mmap_ofs = kMmapSuccess;
+        mmap_ofs = MetaTable::kMmapSuccess;
         event = EVENT_MMAP_SUCCESS_NEW;
       } else if (mmap_ofs > 0) {
         event = EVENT_MMAP_SUCCESS_PARTIAL;
       } else {
-        DCHECK_EQ(kMmapFailure, mmap_ofs);
+        DCHECK_EQ(MetaTable::kMmapFailure, mmap_ofs);
         event = EVENT_MMAP_FAILED_NEW;
       }
 
-      const char* kMmapUpdateStatusSql = "REPLACE INTO meta VALUES (?, ?)";
-      Statement s(GetUniqueStatement(kMmapUpdateStatusSql));
-      s.BindString(0, kMmapStatusKey);
-      s.BindInt64(1, mmap_ofs);
-      if (!s.Run()) {
+      if (!MetaTable::SetMmapStatus(this, mmap_ofs)) {
         RecordOneEvent(EVENT_MMAP_META_FAILURE_UPDATE);
         return 0;
       }
@@ -1011,10 +998,10 @@ size_t Connection::GetAppropriateMmapSize() {
     }
   }
 
-  if (mmap_ofs == kMmapFailure)
+  if (mmap_ofs == MetaTable::kMmapFailure)
     return 0;
-  if (mmap_ofs == kMmapSuccess)
-    return 256 * 1024 * 1024;
+  if (mmap_ofs == MetaTable::kMmapSuccess)
+    return kMmapEverything;
   return mmap_ofs;
 }
 
diff --git a/sql/connection.h b/sql/connection.h
index 18b2cb7..7a80077 100644
--- a/sql/connection.h
+++ b/sql/connection.h
@@ -509,6 +509,7 @@ class SQL_EXPORT Connection : public base::trace_event::MemoryDumpProvider {
   friend class test::ScopedMockTimeSource;
 
   FRIEND_TEST_ALL_PREFIXES(SQLConnectionTest, CollectDiagnosticInfo);
+  FRIEND_TEST_ALL_PREFIXES(SQLConnectionTest, GetAppropriateMmapSize);
   FRIEND_TEST_ALL_PREFIXES(SQLConnectionTest, RegisterIntentToUpload);
 
   // Internal initialize function used by both Init and InitInMemory. The file
diff --git a/sql/connection_unittest.cc b/sql/connection_unittest.cc
index b1946e4..219da7f 100644
--- a/sql/connection_unittest.cc
+++ b/sql/connection_unittest.cc
@@ -1381,4 +1381,91 @@ TEST_F(SQLConnectionTest, RegisterIntentToUpload) {
 }
 #endif  // !defined(MOJO_APPTEST_IMPL)
 
+// Test that a fresh database has mmap enabled by default, if mmap'ed I/O is
+// enabled by SQLite.
+TEST_F(SQLConnectionTest, MmapInitiallyEnabled) {
+  {
+    sql::Statement s(db().GetUniqueStatement("PRAGMA mmap_size"));
+
+    // SQLite doesn't have mmap support (perhaps an early iOS release).
+    if (!s.Step())
+      return;
+
+    // If mmap I/O is not on, attempt to turn it on.  If that succeeds, then
+    // Open() should have turned it on.  If mmap support is disabled, 0 is
+    // returned.  If the VFS does not understand SQLITE_FCNTL_MMAP_SIZE (for
+    // instance MojoVFS), -1 is returned.
+    if (s.ColumnInt(0) <= 0) {
+      ASSERT_TRUE(db().Execute("PRAGMA mmap_size = 1048576"));
+      s.Reset(true);
+      ASSERT_TRUE(s.Step());
+      EXPECT_LE(s.ColumnInt(0), 0);
+    }
+  }
+
+  // Test that explicit disable prevents mmap'ed I/O.
+  db().Close();
+  sql::Connection::Delete(db_path());
+  db().set_mmap_disabled();
+  ASSERT_TRUE(db().Open(db_path()));
+  {
+    sql::Statement s(db().GetUniqueStatement("PRAGMA mmap_size"));
+    ASSERT_TRUE(s.Step());
+    EXPECT_LE(s.ColumnInt(0), 0);
+  }
+}
+
+// Test specific operation of the GetAppropriateMmapSize() helper.
+#if defined(OS_IOS)
+TEST_F(SQLConnectionTest, GetAppropriateMmapSize) {
+  ASSERT_EQ(0UL, db().GetAppropriateMmapSize());
+}
+#else
+TEST_F(SQLConnectionTest, GetAppropriateMmapSize) {
+  const size_t kMmapAlot = 25 * 1024 * 1024;
+
+  // If there is no meta table (as for a fresh database), assume that everything
+  // should be mapped.
+  ASSERT_TRUE(!db().DoesTableExist("meta"));
+  ASSERT_GT(db().GetAppropriateMmapSize(), kMmapAlot);
+
+  // Getting the status fails if there is an error.  GetAppropriateMmapSize()
+  // should not call GetMmapStatus() if the table does not exist, but this is an
+  // easy error to setup for testing.
+  int64_t mmap_status;
+  {
+    sql::ScopedErrorIgnorer ignore_errors;
+    ignore_errors.IgnoreError(SQLITE_ERROR);
+    ASSERT_FALSE(MetaTable::GetMmapStatus(&db(), &mmap_status));
+    ASSERT_TRUE(ignore_errors.CheckIgnoredErrors());
+  }
+
+  // When the meta table is first created, it sets up to map everything.
+  MetaTable().Init(&db(), 1, 1);
+  ASSERT_TRUE(db().DoesTableExist("meta"));
+  ASSERT_GT(db().GetAppropriateMmapSize(), kMmapAlot);
+  ASSERT_TRUE(MetaTable::GetMmapStatus(&db(), &mmap_status));
+  ASSERT_EQ(MetaTable::kMmapSuccess, mmap_status);
+
+  // Failure status maps nothing.
+  ASSERT_TRUE(db().Execute("REPLACE INTO meta VALUES ('mmap_status', -2)"));
+  ASSERT_EQ(0UL, db().GetAppropriateMmapSize());
+
+  // Re-initializing the meta table does not re-create the key if the table
+  // already exists.
+  ASSERT_TRUE(db().Execute("DELETE FROM meta WHERE key = 'mmap_status'"));
+  MetaTable().Init(&db(), 1, 1);
+  ASSERT_EQ(MetaTable::kMmapSuccess, mmap_status);
+  ASSERT_TRUE(MetaTable::GetMmapStatus(&db(), &mmap_status));
+  ASSERT_EQ(0, mmap_status);
+
+  // With no key, map everything and create the key.
+  // TODO(shess): This really should be "maps everything after validating it",
+  // but that is more complicated to structure.
+  ASSERT_GT(db().GetAppropriateMmapSize(), kMmapAlot);
+  ASSERT_TRUE(MetaTable::GetMmapStatus(&db(), &mmap_status));
+  ASSERT_EQ(MetaTable::kMmapSuccess, mmap_status);
+}
+#endif
+
 }  // namespace sql
diff --git a/sql/meta_table.cc b/sql/meta_table.cc
index 319f73d..011fe18 100644
--- a/sql/meta_table.cc
+++ b/sql/meta_table.cc
@@ -13,9 +13,10 @@
 
 namespace {
 
-// Key used in our meta table for version numbers.
+// Keys understood directly by sql:MetaTable.
 const char kVersionKey[] = "version";
 const char kCompatibleVersionKey[] = "last_compatible_version";
+const char kMmapStatusKey[] = "mmap_status";
 
 // Used to track success/failure of deprecation checks.
 enum DeprecationEventType {
@@ -59,12 +60,41 @@ MetaTable::~MetaTable() {
 }
 
 // static
+int64_t MetaTable::kMmapFailure = -2;
+int64_t MetaTable::kMmapSuccess = -1;
+
+// static
 bool MetaTable::DoesTableExist(sql::Connection* db) {
   DCHECK(db);
   return db->DoesTableExist("meta");
 }
 
 // static
+bool MetaTable::GetMmapStatus(Connection* db, int64_t* status) {
+  const char* kMmapStatusSql = "SELECT value FROM meta WHERE key = ?";
+  Statement s(db->GetUniqueStatement(kMmapStatusSql));
+  if (!s.is_valid())
+    return false;
+
+  // It is fine for the status to be missing entirely, but any error prevents
+  // memory-mapping.
+  s.BindString(0, kMmapStatusKey);
+  *status = s.Step() ? s.ColumnInt64(0) : 0;
+  return s.Succeeded();
+}
+
+// static
+bool MetaTable::SetMmapStatus(Connection* db, int64_t status) {
+  DCHECK(status == kMmapFailure || status == kMmapSuccess || status >= 0);
+
+  const char* kMmapUpdateStatusSql = "REPLACE INTO meta VALUES (?, ?)";
+  Statement s(db->GetUniqueStatement(kMmapUpdateStatusSql));
+  s.BindString(0, kMmapStatusKey);
+  s.BindInt64(1, status);
+  return s.Run();
+}
+
+// static
 void MetaTable::RazeIfDeprecated(Connection* db, int deprecated_version) {
   DCHECK_GT(deprecated_version, 0);
   DCHECK_EQ(0, db->transaction_nesting());
@@ -131,6 +161,10 @@ bool MetaTable::Init(Connection* db, int version, int compatible_version) {
         "(key LONGVARCHAR NOT NULL UNIQUE PRIMARY KEY, value LONGVARCHAR)"))
       return false;
 
+    // Newly-created databases start out with mmap'ed I/O, but have no place to
+    // store the setting.  Set here so that later opens don't need to validate.
+    SetMmapStatus(db_, kMmapSuccess);
+
     // Note: there is no index over the meta table. We currently only have a
     // couple of keys, so it doesn't matter. If we start storing more stuff in
     // there, we should create an index.
diff --git a/sql/meta_table.h b/sql/meta_table.h
index 0ae76bd..85c0c82 100644
--- a/sql/meta_table.h
+++ b/sql/meta_table.h
@@ -21,6 +21,13 @@ class SQL_EXPORT MetaTable {
   MetaTable();
   ~MetaTable();
 
+  // Values for Get/SetMmapStatus().  |kMmapFailure| indicates that there was at
+  // some point a read error and the database should not be memory-mapped, while
+  // |kMmapSuccess| indicates that the entire file was read at some point and
+  // can be memory-mapped without constraint.
+  static int64_t kMmapFailure;
+  static int64_t kMmapSuccess;
+
   // Returns true if the 'meta' table exists.
   static bool DoesTableExist(Connection* db);
 
@@ -38,10 +45,18 @@ class SQL_EXPORT MetaTable {
   // transaction.
   static void RazeIfDeprecated(Connection* db, int deprecated_version);
 
+  // Used to tuck some data into the meta table about mmap status.  The value
+  // represents how much data in bytes has successfully been read from the
+  // database, or |kMmapFailure| or |kMmapSuccess|.
+  static bool GetMmapStatus(Connection* db, int64_t* status);
+  static bool SetMmapStatus(Connection* db, int64_t status);
+
   // Initializes the MetaTableHelper, creating the meta table if necessary. For
   // new tables, it will initialize the version number to |version| and the
   // compatible version number to |compatible_version|.  Versions must be
   // greater than 0 to distinguish missing versions (see GetVersionNumber()).
+  // If there was no meta table (proxy for a fresh database), mmap status is set
+  // to |kMmapSuccess|.
   bool Init(Connection* db, int version, int compatible_version);
 
   // Resets this MetaTable object, making another call to Init() possible.