Move blacklisting and pinned URLs from MostVisitedURLs to TopSites.

Prepopulated URLs are also moved into TopSites.
When TopSites becomes the default (without the flag), the corresponding code in MostVisitedURLs can be removed.

BUG=48564, 48566
TEST=TopSitesTest

Committed: http://src.chromium.org/viewvc/chrome?view=rev&revision=53376

Review URL: http://codereview.chromium.org/3036003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@53500 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 670 insertions, 65 deletions

diff --git a/chrome/browser/history/top_sites.cc b/chrome/browser/history/top_sites.cc
index 1ef4dec..d22fc6b 100644
--- a/chrome/browser/history/top_sites.cc
+++ b/chrome/browser/history/top_sites.cc
@@ -6,22 +6,37 @@
 
 #include <algorithm>
 
+#include "app/l10n_util.h"
 #include "base/file_util.h"
 #include "base/logging.h"
+#include "base/md5.h"
+#include "base/string_util.h"
+#include "base/utf_string_conversions.h"
+#include "base/values.h"
 #include "chrome/browser/chrome_thread.h"
-#include "chrome/browser/profile.h"
-#include "chrome/browser/history/top_sites_database.h"
+#include "chrome/browser/dom_ui/most_visited_handler.h"
+#include "chrome/browser/extensions/extensions_service.h"
+#include "chrome/browser/google_util.h"
 #include "chrome/browser/history/history_notifications.h"
 #include "chrome/browser/history/page_usage_data.h"
+#include "chrome/browser/history/top_sites_database.h"
+#include "chrome/browser/pref_service.h"
+#include "chrome/browser/profile.h"
 #include "chrome/browser/tab_contents/navigation_controller.h"
 #include "chrome/browser/tab_contents/navigation_entry.h"
+#include "chrome/common/pref_names.h"
+#include "chrome/common/thumbnail_score.h"
 #include "gfx/codec/jpeg_codec.h"
+#include "grit/chromium_strings.h"
+#include "grit/generated_resources.h"
+#include "grit/locale_settings.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 
 namespace history {
 
 // How many top sites to store in the cache.
 static const size_t kTopSitesNumber = 20;
+static const size_t kTopSitesShown = 8;
 static const int kDaysOfHistory = 90;
 // Time from startup to first HistoryService query.
 static const int64 kUpdateIntervalSecs = 15;
@@ -34,11 +49,18 @@ TopSites::TopSites(Profile* profile) : profile_(profile),
                                        mock_history_service_(NULL),
                                        last_num_urls_changed_(0),
                                        migration_in_progress_(false),
-                                       waiting_for_results_(true) {
+                                       waiting_for_results_(true),
+                                       blacklist_(NULL),
+                                       pinned_urls_(NULL) {
   registrar_.Add(this, NotificationType::HISTORY_URLS_DELETED,
                  Source<Profile>(profile_));
   registrar_.Add(this, NotificationType::NAV_ENTRY_COMMITTED,
                  NotificationService::AllSources());
+
+  blacklist_ = profile_->GetPrefs()->
+      GetMutableDictionary(prefs::kNTPMostVisitedURLsBlacklist);
+  pinned_urls_ = profile_->GetPrefs()->
+      GetMutableDictionary(prefs::kNTPMostVisitedPinnedURLs);
 }
 
 TopSites::~TopSites() {
@@ -70,7 +92,10 @@ void TopSites::ReadDatabase() {
     AutoLock lock(lock_);
     MostVisitedURLList top_urls;
     db_->GetPageThumbnails(&top_urls, &thumbnails);
+    MostVisitedURLList copy(top_urls);  // StoreMostVisited destroys the list.
     StoreMostVisited(&top_urls);
+    if (AddPrepopulatedPages(&copy))
+      UpdateMostVisited(copy);
   }  // Lock is released here.
 
   for (size_t i = 0; i < top_sites_.size(); i++) {
@@ -195,7 +220,6 @@ bool TopSites::SetPageThumbnailNoDB(const GURL& url,
 
 void TopSites::GetMostVisitedURLs(CancelableRequestConsumer* consumer,
                                   GetTopSitesCallback* callback) {
-
   scoped_refptr<CancelableRequest<GetTopSitesCallback> > request(
       new CancelableRequest<GetTopSitesCallback>(callback));
   // This ensures cancelation of requests when either the consumer or the
@@ -209,7 +233,11 @@ void TopSites::GetMostVisitedURLs(CancelableRequestConsumer* consumer,
   }
   if (request->canceled())
     return;
-  request->ForwardResult(GetTopSitesCallback::TupleType(top_sites_));
+
+  MostVisitedURLList filtered_urls;
+  ApplyBlacklistAndPinnedURLs(top_sites_, &filtered_urls);
+
+  request->ForwardResult(GetTopSitesCallback::TupleType(filtered_urls));
 }
 
 bool TopSites::GetPageThumbnail(const GURL& url, RefCountedBytes** data) const {
@@ -222,43 +250,197 @@ bool TopSites::GetPageThumbnail(const GURL& url, RefCountedBytes** data) const {
   return true;
 }
 
-void TopSites::UpdateMostVisited(MostVisitedURLList most_visited) {
-  DCHECK(ChromeThread::CurrentlyOn(ChromeThread::DB));
-  // TODO(brettw) filter for blacklist!
-
-  if (!top_sites_.empty()) {
-    std::vector<size_t> added;    // Indices into most_visited.
-    std::vector<size_t> deleted;  // Indices into top_sites_.
-    std::vector<size_t> moved;    // Indices into most_visited.
-    DiffMostVisited(top_sites_, most_visited, &added, &deleted, &moved);
-
-    // #added == #deleted; #added + #moved = total.
-    last_num_urls_changed_ = added.size() + moved.size();
-
-    // Process the diff: delete from images and disk, add to disk.
-    // Delete all the thumbnails associated with URLs that were deleted.
-    for (size_t i = 0; i < deleted.size(); i++) {
-      const MostVisitedURL& deleted_url = top_sites_[deleted[i]];
-      std::map<GURL, Images>::iterator found =
-          top_images_.find(deleted_url.url);
-      if (found != top_images_.end())
-        top_images_.erase(found);
-
-      // Delete from disk.
-      if (db_.get())
-        db_->RemoveURL(deleted_url);
+static int IndexOf(const MostVisitedURLList& urls, const GURL& url) {
+  for (size_t i = 0; i < urls.size(); i++) {
+    if (urls[i].url == url)
+      return i;
+  }
+  return -1;
+}
+
+int TopSites::GetIndexForChromeStore(const MostVisitedURLList& urls) {
+  GURL store_url = MostVisitedHandler::GetChromeStoreURLWithLocale();
+  if (IsBlacklisted(store_url))
+    return -1;
+
+  if (IndexOf(urls, store_url) != -1)
+    return -1;  // It's already there, no need to add.
+
+  // Should replace the first filler.
+  int first_filler = IndexOf(urls, GURL());
+  if (first_filler != -1)
+    return first_filler;
+
+  if (urls.size() < kTopSitesShown)
+    return urls.size();
+
+  // Should replace the last non-pinned url.
+  for (size_t i = kTopSitesShown - 1; i < urls.size(); i--) {
+    if (!IsURLPinned(urls[i].url))
+      return i;
+  }
+
+  // All urls are pinned.
+  return -1;
+}
+
+bool TopSites::AddChromeStore(MostVisitedURLList* urls) {
+  ExtensionsService* service = profile_->GetExtensionsService();
+  if (!service || service->HasApps())
+    return false;
+
+  int index = GetIndexForChromeStore(*urls);
+  if (index == -1)
+    return false;
+
+  if (static_cast<size_t>(index) >= urls->size())
+    urls->resize(index + 1);
+
+  // Chrome App store may replace an existing non-pinned thumbnail.
+  MostVisitedURL& url = (*urls)[index];
+  url.url = MostVisitedHandler::GetChromeStoreURLWithLocale();
+  url.title = l10n_util::GetStringUTF16(IDS_EXTENSION_WEB_STORE_TITLE);
+  url.favicon_url =
+      GURL("chrome://theme/IDR_NEWTAB_CHROME_STORE_PAGE_FAVICON");
+  url.redirects.push_back(url.url);
+  return true;
+}
+
+bool TopSites::AddPrepopulatedPages(MostVisitedURLList* urls) {
+  // TODO(arv): This needs to get the data from some configurable place.
+  // http://crbug.com/17630
+  bool added = false;
+  GURL welcome_url(WideToUTF8(l10n_util::GetString(IDS_CHROME_WELCOME_URL)));
+  if (urls->size() < kTopSitesNumber && IndexOf(*urls, welcome_url) == -1) {
+    MostVisitedURL url = {
+      welcome_url,
+      GURL("chrome://theme/IDR_NEWTAB_CHROME_WELCOME_PAGE_FAVICON"),
+      l10n_util::GetStringUTF16(IDS_NEW_TAB_CHROME_WELCOME_PAGE_TITLE)
+    };
+    url.redirects.push_back(welcome_url);
+    urls->push_back(url);
+    added = true;
+  }
+
+  GURL themes_url(WideToUTF8(l10n_util::GetString(IDS_THEMES_GALLERY_URL)));
+  if (urls->size() < kTopSitesNumber && IndexOf(*urls, themes_url) == -1) {
+    MostVisitedURL url = {
+      themes_url,
+      GURL("chrome://theme/IDR_NEWTAB_THEMES_GALLERY_FAVICON"),
+      l10n_util::GetStringUTF16(IDS_NEW_TAB_THEMES_GALLERY_PAGE_TITLE)
+    };
+    url.redirects.push_back(themes_url);
+    urls->push_back(url);
+    added = true;
+  }
+
+  if (AddChromeStore(urls))
+    added = true;
+
+  return added;
+}
+
+void TopSites::MigratePinnedURLs() {
+  std::map<GURL, size_t> tmp_map;
+  for (DictionaryValue::key_iterator it = pinned_urls_->begin_keys();
+       it != pinned_urls_->end_keys(); ++it) {
+    Value* value;
+    if (!pinned_urls_->GetWithoutPathExpansion(*it, &value))
+      continue;
+
+    if (value->IsType(DictionaryValue::TYPE_DICTIONARY)) {
+      DictionaryValue* dict = static_cast<DictionaryValue*>(value);
+      std::string url_string;
+      int index;
+      if (dict->GetString(L"url", &url_string) &&
+          dict->GetInteger(L"index", &index))
+        tmp_map[GURL(url_string)] = index;
     }
+  }
+  pinned_urls_->Clear();
+  for (std::map<GURL, size_t>::iterator it = tmp_map.begin();
+       it != tmp_map.end(); ++it)
+    AddPinnedURL(it->first, it->second);
+}
 
-    if (db_.get()) {
-      // Write both added and moved urls.
-      for (size_t i = 0; i < added.size(); i++) {
-        MostVisitedURL& added_url = most_visited[added[i]];
-        db_->SetPageThumbnail(added_url, added[i], Images());
-      }
-      for (size_t i = 0; i < moved.size(); i++) {
-        MostVisitedURL moved_url = most_visited[moved[i]];
-        db_->UpdatePageRank(moved_url, moved[i]);
+void TopSites::ApplyBlacklistAndPinnedURLs(const MostVisitedURLList& urls,
+                                           MostVisitedURLList* out) {
+  for (size_t i = 0; i < urls.size(); i++) {
+    if (!IsBlacklisted(urls[i].url))
+      out->push_back(urls[i]);
+  }
+
+  for (DictionaryValue::key_iterator it = pinned_urls_->begin_keys();
+       it != pinned_urls_->end_keys(); ++it) {
+    GURL url(WideToASCII(*it));
+
+    int index = IndexOf(*out, url);
+    if (index == -1) {
+      if (url.is_empty()) {
+        MigratePinnedURLs();
+        out->clear();
+        ApplyBlacklistAndPinnedURLs(urls, out);
+        return;
       }
+      LOG(INFO) << "Unknown url: " << url.spec();
+      continue;
+    }
+
+    size_t pinned_index = 0;
+    bool result = GetIndexOfPinnedURL(url, &pinned_index);
+    DCHECK(result) << url.spec();
+    if (static_cast<int>(pinned_index) != index) {
+      MostVisitedURL tmp = (*out)[index];
+      out->erase(out->begin() + index);
+      if (pinned_index > out->size())
+        out->resize(pinned_index);  // Add empty URLs as fillers.
+      out->insert(out->begin() + pinned_index, tmp);
+    }
+  }
+}
+
+std::wstring TopSites::GetURLString(const GURL& url) {
+  return ASCIIToWide(GetCanonicalURL(url).spec());
+}
+
+std::wstring TopSites::GetURLHash(const GURL& url) {
+  return ASCIIToWide(MD5String(GetCanonicalURL(url).spec()));
+}
+
+void TopSites::UpdateMostVisited(MostVisitedURLList most_visited) {
+  DCHECK(ChromeThread::CurrentlyOn(ChromeThread::DB));
+
+  std::vector<size_t> added;    // Indices into most_visited.
+  std::vector<size_t> deleted;  // Indices into top_sites_.
+  std::vector<size_t> moved;    // Indices into most_visited.
+  DiffMostVisited(top_sites_, most_visited, &added, &deleted, &moved);
+
+  // #added == #deleted; #added + #moved = total.
+  last_num_urls_changed_ = added.size() + moved.size();
+
+  // Process the diff: delete from images and disk, add to disk.
+  // Delete all the thumbnails associated with URLs that were deleted.
+  for (size_t i = 0; i < deleted.size(); i++) {
+    const MostVisitedURL& deleted_url = top_sites_[deleted[i]];
+    std::map<GURL, Images>::iterator found =
+        top_images_.find(deleted_url.url);
+    if (found != top_images_.end())
+      top_images_.erase(found);
+
+    // Delete from disk.
+    if (db_.get())
+      db_->RemoveURL(deleted_url);
+  }
+
+  if (db_.get()) {
+    // Write both added and moved urls.
+    for (size_t i = 0; i < added.size(); i++) {
+      const MostVisitedURL& added_url = most_visited[added[i]];
+      db_->SetPageThumbnail(added_url, added[i], Images());
+    }
+    for (size_t i = 0; i < moved.size(); i++) {
+      const MostVisitedURL& moved_url = most_visited[moved[i]];
+      db_->UpdatePageRank(moved_url, moved[i]);
     }
   }
 
@@ -302,6 +484,12 @@ void TopSites::AddTemporaryThumbnail(const GURL& url,
 
 void TopSites::StartQueryForThumbnail(size_t index) {
   DCHECK(migration_in_progress_);
+  if (top_sites_[index].url.spec() ==
+      l10n_util::GetStringUTF8(IDS_CHROME_WELCOME_URL) ||
+      top_sites_[index].url.spec() ==
+      l10n_util::GetStringUTF8(IDS_THEMES_GALLERY_URL))
+    return;  // Don't need thumbnails for prepopulated URLs.
+
   migration_pending_urls_.insert(top_sites_[index].url);
 
   if (mock_history_service_) {
@@ -374,7 +562,7 @@ void TopSites::StoreRedirectChain(const RedirectList& redirects,
 GURL TopSites::GetCanonicalURL(const GURL& url) const {
   std::map<GURL, size_t>::const_iterator found = canonical_urls_.find(url);
   if (found == canonical_urls_.end())
-    return GURL();  // Don't know anything about this URL.
+    return url;  // Unknown URL - return unchanged.
   return top_sites_[found->second].url;
 }
 
@@ -459,6 +647,75 @@ void TopSites::StartMigration() {
   StartQueryForMostVisited();
 }
 
+void TopSites::AddBlacklistedURL(const GURL& url) {
+  RemovePinnedURL(url);
+  Value* dummy = Value::CreateNullValue();
+  blacklist_->SetWithoutPathExpansion(GetURLHash(url), dummy);
+}
+
+bool TopSites::IsBlacklisted(const GURL& url) {
+  Value* dummy = Value::CreateNullValue();
+  bool result = blacklist_->GetWithoutPathExpansion(GetURLHash(url), &dummy);
+  return result;
+}
+
+void TopSites::RemoveBlacklistedURL(const GURL& url) {
+  Value* dummy = NULL;
+  blacklist_->RemoveWithoutPathExpansion(GetURLHash(url), &dummy);
+}
+
+void TopSites::ClearBlacklistedURLs() {
+  blacklist_->Clear();
+}
+
+void TopSites::AddPinnedURL(const GURL& url, size_t pinned_index) {
+  GURL old;
+  if (GetPinnedURLAtIndex(pinned_index, &old)) {
+    RemovePinnedURL(old);
+  }
+
+  if (IsURLPinned(url)) {
+    RemovePinnedURL(url);
+  }
+
+  Value* index = Value::CreateIntegerValue(pinned_index);
+  pinned_urls_->SetWithoutPathExpansion(GetURLString(url), index);
+}
+
+void TopSites::RemovePinnedURL(const GURL& url) {
+  Value* dummy = NULL;
+  pinned_urls_->RemoveWithoutPathExpansion(GetURLString(url), &dummy);
+}
+
+bool TopSites::GetIndexOfPinnedURL(const GURL& url, size_t* index) {
+  int tmp;
+  bool result = pinned_urls_->GetIntegerWithoutPathExpansion(
+      GetURLString(url), &tmp);
+  *index = static_cast<size_t>(tmp);
+  return result;
+}
+
+bool TopSites::IsURLPinned(const GURL& url) {
+  int tmp;
+  bool result = pinned_urls_->GetIntegerWithoutPathExpansion(
+      GetURLString(url), &tmp);
+  return result;
+}
+
+bool TopSites::GetPinnedURLAtIndex(size_t index, GURL* url) {
+  for (DictionaryValue::key_iterator it = pinned_urls_->begin_keys();
+       it != pinned_urls_->end_keys(); ++it) {
+    int current_index;
+    if (pinned_urls_->GetIntegerWithoutPathExpansion(*it, &current_index)) {
+      if (static_cast<size_t>(current_index) == index) {
+        *url = GURL(WideToASCII(*it));
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 base::TimeDelta TopSites::GetUpdateDelay() {
   if (top_sites_.size() == 0)
     return base::TimeDelta::FromSeconds(30);
@@ -472,20 +729,24 @@ base::TimeDelta TopSites::GetUpdateDelay() {
 void TopSites::OnTopSitesAvailable(
     CancelableRequestProvider::Handle handle,
     MostVisitedURLList pages) {
+
+  AddPrepopulatedPages(&pages);
+  ChromeThread::PostTask(ChromeThread::DB, FROM_HERE, NewRunnableMethod(
+      this, &TopSites::UpdateMostVisited, pages));
+
   if (!pending_callbacks_.empty()) {
-    PendingCallbackSet copy(pending_callbacks_);
+    MostVisitedURLList filtered_urls;
+    ApplyBlacklistAndPinnedURLs(pages, &filtered_urls);
+
     PendingCallbackSet::iterator i;
     for (i = pending_callbacks_.begin();
          i != pending_callbacks_.end(); ++i) {
       scoped_refptr<CancelableRequest<GetTopSitesCallback> > request = *i;
       if (!request->canceled())
-        request->ForwardResult(GetTopSitesCallback::TupleType(pages));
+        request->ForwardResult(GetTopSitesCallback::TupleType(filtered_urls));
     }
     pending_callbacks_.clear();
   }
-
-  ChromeThread::PostTask(ChromeThread::DB, FROM_HERE, NewRunnableMethod(
-      this, &TopSites::UpdateMostVisited, pages));
 }
 
 void TopSites::OnThumbnailAvailable(CancelableRequestProvider::Handle handle,
diff --git a/chrome/browser/history/top_sites.h b/chrome/browser/history/top_sites.h
index 4cc6059a..e3bb1ab 100644
--- a/chrome/browser/history/top_sites.h
+++ b/chrome/browser/history/top_sites.h
@@ -24,6 +24,7 @@
 #include "chrome/common/thumbnail_score.h"
 #include "googleurl/src/gurl.h"
 
+class DictionaryValue;
 class SkBitmap;
 class Profile;
 
@@ -101,6 +102,38 @@ class TopSites : public NotificationObserver,
   // Start reading thumbnails from the ThumbnailDatabase.
   void StartMigration();
 
+  // Blacklisted URLs
+
+  // Add a  URL to the blacklist.
+  void AddBlacklistedURL(const GURL& url);
+
+  // Returns true if the URL is blacklisted.
+  bool IsBlacklisted(const GURL& url);
+
+  // Removes a URL from the blacklist.
+  void RemoveBlacklistedURL(const GURL& url);
+
+  // Clear the blacklist.
+  void ClearBlacklistedURLs();
+
+  // Pinned URLs
+
+  // Pin a URL at |index|.
+  void AddPinnedURL(const GURL& url, size_t index);
+
+  // Get the index of a URL. Returns true if |url| is pinned.
+  bool GetIndexOfPinnedURL(const GURL& url, size_t* index);
+
+  // Returns true if a URL is pinned.
+  bool IsURLPinned(const GURL& url);
+
+  // Unpin a URL.
+  void RemovePinnedURL(const GURL& url);
+
+  // Return a URL pinned at |index| via |out|. Returns true if there
+  // is a URL pinned at |index|.
+  bool GetPinnedURLAtIndex(size_t index, GURL* out);
+
  private:
   friend class base::RefCountedThreadSafe<TopSites>;
   friend class TopSitesTest;
@@ -113,6 +146,11 @@ class TopSites : public NotificationObserver,
   FRIEND_TEST_ALL_PREFIXES(TopSitesTest, QueueingRequestsForTopSites);
   FRIEND_TEST_ALL_PREFIXES(TopSitesTest, CancelingRequestsForTopSites);
   FRIEND_TEST_ALL_PREFIXES(TopSitesTest, AddTemporaryThumbnail);
+  FRIEND_TEST_ALL_PREFIXES(TopSitesTest, Blacklisting);
+  FRIEND_TEST_ALL_PREFIXES(TopSitesTest, PinnedURLs);
+  FRIEND_TEST_ALL_PREFIXES(TopSitesTest, BlacklistingAndPinnedURLs);
+  FRIEND_TEST_ALL_PREFIXES(TopSitesTest, AddPrepopulatedPages);
+  FRIEND_TEST_ALL_PREFIXES(TopSitesTest, GetIndexForChromeStore);
 
   ~TopSites();
 
@@ -216,6 +254,33 @@ class TopSites : public NotificationObserver,
                              const RefCountedBytes* thumbnail,
                              const ThumbnailScore& score);
 
+  // Returns an index of a thumbnail that should be replaced by the
+  // Chrome App Store. Returns -1 App Store should not be added.
+  int GetIndexForChromeStore(const MostVisitedURLList& urls);
+
+  // Adds Chrome App Store thumbnail to a list of URLs, if possible.
+  // Returns true if it was added.
+  bool AddChromeStore(MostVisitedURLList* urls);
+
+  // Add prepopulated pages: 'welcome to Chrome' and themes gallery.
+  // Returns true if any pages were added.
+  bool AddPrepopulatedPages(MostVisitedURLList* urls);
+
+  // Convert pinned_urls_ dictionary to the new format. Use URLs as
+  // dictionary keys.
+  void MigratePinnedURLs();
+
+  // Takes |urls|, produces it's copy in |out| after removing
+  // blacklisted URLs and reordering pinned URLs.
+  void ApplyBlacklistAndPinnedURLs(const MostVisitedURLList& urls,
+                                   MostVisitedURLList* out);
+
+  // Converts a url into a canonical string representation.
+  std::wstring GetURLString(const GURL& url);
+
+  // Returns an MD5 hash of the URL. Hashing is required for blacklisted URLs.
+  std::wstring GetURLHash(const GURL& url);
+
   Profile* profile_;
   // A mockup to use for testing. If NULL, use the real HistoryService
   // from the profile_. See SetMockHistoryService.
@@ -270,8 +335,20 @@ class TopSites : public NotificationObserver,
   // UpdateMostVisitedURLs call.
   std::map<GURL, Images> temp_thumbnails_map_;
 
-  // TODO(brettw): use the blacklist.
-  // std::set<GURL> blacklist_;
+  // Blacklisted and pinned URLs are stored in Preferences.
+
+  // Blacklisted URLs. They are filtered out from the list of Top
+  // Sites when GetMostVisitedURLs is called. Note that we are still
+  // storing all URLs, but filtering on access. It is a dictionary,
+  // key is the URL, value is a dummy value. This is owned by the
+  // PrefService.
+  DictionaryValue* blacklist_;
+
+  // This is a dictionary for the pinned URLs for the the most visited
+  // part of the new tab page. Key is the URL, value is
+  // index where it is pinned at (may be the same as key). This is
+  // owned by the PrefService.
+  DictionaryValue* pinned_urls_;
 
   DISALLOW_COPY_AND_ASSIGN(TopSites);
 };
diff --git a/chrome/browser/history/top_sites_unittest.cc b/chrome/browser/history/top_sites_unittest.cc
index 8d0d81e..60d4a86 100644
--- a/chrome/browser/history/top_sites_unittest.cc
+++ b/chrome/browser/history/top_sites_unittest.cc
@@ -2,10 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+#include "app/l10n_util.h"
 #include "base/scoped_temp_dir.h"
 #include "base/string_util.h"
 #include "chrome/browser/history/top_sites.h"
 #include "chrome/common/chrome_paths.h"
+#include "chrome/browser/dom_ui/most_visited_handler.h"
 #include "chrome/browser/history/history_marshaling.h"
 #include "chrome/browser/history/top_sites_database.h"
 #include "chrome/browser/history/history_notifications.h"
@@ -13,6 +15,9 @@
 #include "chrome/tools/profiles/thumbnail-inl.h"
 #include "gfx/codec/jpeg_codec.h"
 #include "googleurl/src/gurl.h"
+#include "grit/chromium_strings.h"
+#include "grit/generated_resources.h"
+#include "grit/locale_settings.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 
@@ -38,6 +43,13 @@ class TopSitesTest : public testing::Test {
   RefCountedBytes* weewar_thumbnail() { return weewar_thumbnail_; }
   CancelableRequestConsumer* consumer() { return &consumer_; }
   size_t number_of_callbacks() {return number_of_callbacks_; }
+  // Prepopulated URLs - added at the back of TopSites.
+  GURL welcome_url() {
+    return GURL(WideToUTF8(l10n_util::GetString(IDS_CHROME_WELCOME_URL)));
+  }
+  GURL themes_url() {
+    return GURL(WideToUTF8(l10n_util::GetString(IDS_THEMES_GALLERY_URL)));
+  }
 
   virtual void SetUp() {
     profile_.reset(new TestingProfile);
@@ -151,7 +163,7 @@ class MockHistoryServiceImpl : public TopSites::MockHistoryService {
     MostVisitedURLList::iterator pos = std::find(most_visited_urls_.begin(),
                                                  most_visited_urls_.end(),
                                                  mvu);
-    EXPECT_TRUE(pos != most_visited_urls_.end());
+    EXPECT_TRUE(pos != most_visited_urls_.end()) << url.spec();
     scoped_refptr<RefCountedBytes> thumbnail;
     callback->Run(index, thumbnail);
     delete callback;
@@ -294,9 +306,9 @@ TEST_F(TopSitesTest, GetCanonicalURL) {
   AppendMostVisitedURL(&most_visited, news);
   StoreMostVisited(&most_visited);
 
-  // Random URLs not in the database shouldn't be reported as being in there.
+  // Random URLs not in the database are returned unchanged.
   GURL result = GetCanonicalURL(GURL("http://fark.com/"));
-  EXPECT_TRUE(result.is_empty());
+  EXPECT_EQ(GURL("http://fark.com/"), result);
 
   // Easy case, there are no redirects and the exact URL is stored.
   result = GetCanonicalURL(news);
@@ -411,9 +423,12 @@ TEST_F(TopSitesTest, GetMostVisited) {
       consumer(),
       NewCallback(static_cast<TopSitesTest*>(this),
                   &TopSitesTest::OnTopSitesAvailable));
-  ASSERT_EQ(2u, urls().size());
+  // 2 extra prepopulated URLs.
+  ASSERT_EQ(4u, urls().size());
   EXPECT_EQ(news, urls()[0].url);
   EXPECT_EQ(google, urls()[1].url);
+  EXPECT_EQ(welcome_url(), urls()[2].url);
+  EXPECT_EQ(themes_url(), urls()[3].url);
 }
 
 TEST_F(TopSitesTest, MockDatabase) {
@@ -441,9 +456,11 @@ TEST_F(TopSitesTest, MockDatabase) {
       consumer(),
       NewCallback(static_cast<TopSitesTest*>(this),
                   &TopSitesTest::OnTopSitesAvailable));
-  ASSERT_EQ(1u, urls().size());
+  ASSERT_EQ(3u, urls().size());
   EXPECT_EQ(asdf_url, urls()[0].url);
   EXPECT_EQ(asdf_title, urls()[0].title);
+  EXPECT_EQ(welcome_url(), urls()[1].url);
+  EXPECT_EQ(themes_url(), urls()[2].url);
 
   MostVisitedURL url2;
   url2.url = google_url;
@@ -459,11 +476,13 @@ TEST_F(TopSitesTest, MockDatabase) {
       consumer(),
       NewCallback(static_cast<TopSitesTest*>(this),
                   &TopSitesTest::OnTopSitesAvailable));
-  ASSERT_EQ(2u, urls().size());
+  ASSERT_EQ(4u, urls().size());
   EXPECT_EQ(google_url, urls()[0].url);
   EXPECT_EQ(google_title, urls()[0].title);
   EXPECT_EQ(asdf_url, urls()[1].url);
   EXPECT_EQ(asdf_title, urls()[1].title);
+  EXPECT_EQ(welcome_url(), urls()[2].url);
+  EXPECT_EQ(themes_url(), urls()[3].url);
 
   MockHistoryServiceImpl hs;
   // Add one old, one new URL to the history.
@@ -478,7 +497,7 @@ TEST_F(TopSitesTest, MockDatabase) {
   std::map<GURL, TopSites::Images> thumbnails;
   MostVisitedURLList result;
   db->GetPageThumbnails(&result, &thumbnails);
-  ASSERT_EQ(2u, result.size());
+  ASSERT_EQ(4u, result.size());
   EXPECT_EQ(google_title, result[0].title);
   EXPECT_EQ(news_title, result[1].title);
 }
@@ -597,9 +616,11 @@ TEST_F(TopSitesTest, RealDatabase) {
       consumer(),
       NewCallback(static_cast<TopSitesTest*>(this),
                   &TopSitesTest::OnTopSitesAvailable));
-  ASSERT_EQ(1u, urls().size());
+  ASSERT_EQ(3u, urls().size());
   EXPECT_EQ(asdf_url, urls()[0].url);
   EXPECT_EQ(asdf_title, urls()[0].title);
+  EXPECT_EQ(welcome_url(), urls()[1].url);
+  EXPECT_EQ(themes_url(), urls()[2].url);
 
   TopSites::Images img_result;
   db->GetPageThumbnail(asdf_url, &img_result);
@@ -629,7 +650,7 @@ TEST_F(TopSitesTest, RealDatabase) {
       consumer(),
       NewCallback(static_cast<TopSitesTest*>(this),
                   &TopSitesTest::OnTopSitesAvailable));
-  ASSERT_EQ(2u, urls().size());
+  ASSERT_EQ(4u, urls().size());
   EXPECT_EQ(google1_url, urls()[0].url);
   EXPECT_EQ(google_title, urls()[0].title);
   EXPECT_TRUE(top_sites().GetPageThumbnail(google1_url, &thumbnail_result));
@@ -641,6 +662,8 @@ TEST_F(TopSitesTest, RealDatabase) {
 
   EXPECT_EQ(asdf_url, urls()[1].url);
   EXPECT_EQ(asdf_title, urls()[1].title);
+  EXPECT_EQ(welcome_url(), urls()[2].url);
+  EXPECT_EQ(themes_url(), urls()[3].url);
 
   MockHistoryServiceImpl hs;
   // Add one old, one new URL to the history.
@@ -655,7 +678,7 @@ TEST_F(TopSitesTest, RealDatabase) {
   std::map<GURL, TopSites::Images> thumbnails;
   MostVisitedURLList results;
   db->GetPageThumbnails(&results, &thumbnails);
-  ASSERT_EQ(2u, results.size());
+  ASSERT_EQ(4u, results.size());
   EXPECT_EQ(google_title, results[0].title);
   EXPECT_EQ(news_title, results[1].title);
 
@@ -729,7 +752,8 @@ TEST_F(TopSitesTest, DeleteNotifications) {
       consumer(),
       NewCallback(static_cast<TopSitesTest*>(this),
                   &TopSitesTest::OnTopSitesAvailable));
-  ASSERT_EQ(2u, urls().size());
+  // 2 extra prepopulated URLs.
+  ASSERT_EQ(4u, urls().size());
 
   hs.RemoveMostVisitedURL();
 
@@ -745,8 +769,10 @@ TEST_F(TopSitesTest, DeleteNotifications) {
       consumer(),
       NewCallback(static_cast<TopSitesTest*>(this),
                   &TopSitesTest::OnTopSitesAvailable));
-  ASSERT_EQ(1u, urls().size());
+  ASSERT_EQ(3u, urls().size());
   EXPECT_EQ(google_title, urls()[0].title);
+  EXPECT_EQ(welcome_url(), urls()[1].url);
+  EXPECT_EQ(themes_url(), urls()[2].url);
 
   hs.RemoveMostVisitedURL();
   history_details.all_history = true;
@@ -759,7 +785,9 @@ TEST_F(TopSitesTest, DeleteNotifications) {
       consumer(),
       NewCallback(static_cast<TopSitesTest*>(this),
                   &TopSitesTest::OnTopSitesAvailable));
-  ASSERT_EQ(0u, urls().size());
+  ASSERT_EQ(2u, urls().size());
+  EXPECT_EQ(welcome_url(), urls()[0].url);
+  EXPECT_EQ(themes_url(), urls()[1].url);
 }
 
 TEST_F(TopSitesTest, GetUpdateDelay) {
@@ -780,8 +808,6 @@ TEST_F(TopSitesTest, GetUpdateDelay) {
 TEST_F(TopSitesTest, Migration) {
   ChromeThread db_loop(ChromeThread::DB, MessageLoop::current());
   GURL google1_url("http://google.com");
-  GURL google2_url("http://google.com/redirect");
-  GURL google3_url("http://www.google.com");
   string16 google_title(ASCIIToUTF16("Google"));
   GURL news_url("http://news.google.com");
   string16 news_title(ASCIIToUTF16("Google News"));
@@ -793,6 +819,7 @@ TEST_F(TopSitesTest, Migration) {
   hs.AppendMockPage(google1_url, google_title);
   hs.AppendMockPage(news_url, news_title);
   top_sites().SetMockHistoryService(&hs);
+  MessageLoop::current()->RunAllPending();
 
   top_sites().StartMigration();
   EXPECT_TRUE(top_sites().migration_in_progress_);
@@ -837,9 +864,12 @@ TEST_F(TopSitesTest, QueueingRequestsForTopSites) {
 
   EXPECT_EQ(3u, number_of_callbacks());
 
-  ASSERT_EQ(2u, urls().size());
+  ASSERT_EQ(4u, urls().size());
   EXPECT_EQ("http://1.com/", urls()[0].url.spec());
   EXPECT_EQ("http://2.com/", urls()[1].url.spec());
+  EXPECT_EQ(welcome_url(), urls()[2].url);
+  EXPECT_EQ(themes_url(), urls()[3].url);
+
 
   url.url = GURL("http://3.com/");
   url.redirects.push_back(url.url);
@@ -854,10 +884,13 @@ TEST_F(TopSitesTest, QueueingRequestsForTopSites) {
 
   EXPECT_EQ(4u, number_of_callbacks());
 
-  ASSERT_EQ(3u, urls().size());
+  ASSERT_EQ(5u, urls().size());
   EXPECT_EQ("http://1.com/", urls()[0].url.spec());
   EXPECT_EQ("http://2.com/", urls()[1].url.spec());
   EXPECT_EQ("http://3.com/", urls()[2].url.spec());
+  EXPECT_EQ(welcome_url(), urls()[3].url);
+  EXPECT_EQ(themes_url(), urls()[4].url);
+
 }
 
 TEST_F(TopSitesTest, CancelingRequestsForTopSites) {
@@ -899,7 +932,8 @@ TEST_F(TopSitesTest, CancelingRequestsForTopSites) {
   // 1 request was canceled.
   EXPECT_EQ(2u, number_of_callbacks());
 
-  ASSERT_EQ(2u, urls().size());
+  // 2 extra prepopulated URLs.
+  ASSERT_EQ(4u, urls().size());
   EXPECT_EQ("http://1.com/", urls()[0].url.spec());
   EXPECT_EQ("http://2.com/", urls()[1].url.spec());
 }
@@ -947,4 +981,237 @@ TEST_F(TopSitesTest, AddTemporaryThumbnail) {
                       thumbnail.getSize()));
 }
 
+TEST_F(TopSitesTest, Blacklisting) {
+  ChromeThread db_loop(ChromeThread::DB, MessageLoop::current());
+  MostVisitedURLList pages;
+  MostVisitedURL url, url1;
+  url.url = GURL("http://bbc.com/");
+  url.redirects.push_back(url.url);
+  pages.push_back(url);
+  url1.url = GURL("http://google.com/");
+  url1.redirects.push_back(url1.url);
+  pages.push_back(url1);
+
+  CancelableRequestConsumer c;
+  top_sites().GetMostVisitedURLs(
+      &c,
+      NewCallback(static_cast<TopSitesTest*>(this),
+                  &TopSitesTest::OnTopSitesAvailable));
+  top_sites().OnTopSitesAvailable(0, pages);
+  MessageLoop::current()->RunAllPending();
+  EXPECT_FALSE(top_sites().IsBlacklisted(GURL("http://bbc.com/")));
+
+  EXPECT_EQ(1u, number_of_callbacks());
+
+  ASSERT_EQ(4u, urls().size());
+  EXPECT_EQ("http://bbc.com/", urls()[0].url.spec());
+  EXPECT_EQ("http://google.com/", urls()[1].url.spec());
+  EXPECT_EQ(welcome_url(), urls()[2].url);
+  EXPECT_EQ(themes_url(), urls()[3].url);
+
+  top_sites().AddBlacklistedURL(GURL("http://google.com/"));
+  EXPECT_TRUE(top_sites().IsBlacklisted(GURL("http://google.com/")));
+  EXPECT_FALSE(top_sites().IsBlacklisted(GURL("http://bbc.com/")));
+  EXPECT_FALSE(top_sites().IsBlacklisted(welcome_url()));
+
+  top_sites().GetMostVisitedURLs(
+      &c,
+      NewCallback(static_cast<TopSitesTest*>(this),
+                  &TopSitesTest::OnTopSitesAvailable));
+  MessageLoop::current()->RunAllPending();
+  EXPECT_EQ(2u, number_of_callbacks());
+  ASSERT_EQ(3u, urls().size());
+  EXPECT_EQ("http://bbc.com/", urls()[0].url.spec());
+  EXPECT_EQ(welcome_url(), urls()[1].url);
+  EXPECT_EQ(themes_url(), urls()[2].url);
+
+  top_sites().AddBlacklistedURL(welcome_url());
+  top_sites().GetMostVisitedURLs(
+      &c,
+      NewCallback(static_cast<TopSitesTest*>(this),
+                  &TopSitesTest::OnTopSitesAvailable));
+  ASSERT_EQ(2u, urls().size());
+  EXPECT_EQ("http://bbc.com/", urls()[0].url.spec());
+  EXPECT_EQ(themes_url(), urls()[1].url);
+
+  top_sites().RemoveBlacklistedURL(GURL("http://google.com/"));
+  EXPECT_FALSE(top_sites().IsBlacklisted(GURL("http://google.com/")));
+
+  top_sites().GetMostVisitedURLs(
+      &c,
+      NewCallback(static_cast<TopSitesTest*>(this),
+                  &TopSitesTest::OnTopSitesAvailable));
+  ASSERT_EQ(3u, urls().size());
+  EXPECT_EQ("http://bbc.com/", urls()[0].url.spec());
+  EXPECT_EQ("http://google.com/", urls()[1].url.spec());
+  EXPECT_EQ(themes_url(), urls()[2].url);
+
+  top_sites().ClearBlacklistedURLs();
+  top_sites().GetMostVisitedURLs(
+      &c,
+      NewCallback(static_cast<TopSitesTest*>(this),
+                  &TopSitesTest::OnTopSitesAvailable));
+  ASSERT_EQ(4u, urls().size());
+  EXPECT_EQ("http://bbc.com/", urls()[0].url.spec());
+  EXPECT_EQ("http://google.com/", urls()[1].url.spec());
+  EXPECT_EQ(welcome_url(), urls()[2].url);
+  EXPECT_EQ(themes_url(), urls()[3].url);
+}
+
+TEST_F(TopSitesTest, PinnedURLs) {
+  ChromeThread db_loop(ChromeThread::DB, MessageLoop::current());
+  MostVisitedURLList pages;
+  MostVisitedURL url, url1;
+  url.url = GURL("http://bbc.com/");
+  url.redirects.push_back(url.url);
+  pages.push_back(url);
+  url1.url = GURL("http://google.com/");
+  url1.redirects.push_back(url1.url);
+  pages.push_back(url1);
+
+  CancelableRequestConsumer c;
+  top_sites().GetMostVisitedURLs(
+      &c,
+      NewCallback(static_cast<TopSitesTest*>(this),
+                  &TopSitesTest::OnTopSitesAvailable));
+  top_sites().OnTopSitesAvailable(0, pages);
+  MessageLoop::current()->RunAllPending();
+  size_t index = 0;
+  EXPECT_FALSE(top_sites().IsURLPinned(GURL("http://bbc.com/")));
+
+  ASSERT_EQ(4u, urls().size());
+  EXPECT_EQ("http://bbc.com/", urls()[0].url.spec());
+  EXPECT_EQ("http://google.com/", urls()[1].url.spec());
+  EXPECT_EQ(welcome_url(), urls()[2].url);
+  EXPECT_EQ(themes_url(), urls()[3].url);
+
+  top_sites().AddPinnedURL(GURL("http://google.com/"), 3);
+  EXPECT_TRUE(top_sites().GetIndexOfPinnedURL(GURL("http://google.com/"),
+                                              &index));
+  EXPECT_EQ(3u, index);
+  EXPECT_FALSE(top_sites().IsURLPinned(GURL("http://bbc.com/")));
+  EXPECT_FALSE(top_sites().IsURLPinned(welcome_url()));
+
+  top_sites().GetMostVisitedURLs(
+      &c,
+      NewCallback(static_cast<TopSitesTest*>(this),
+                  &TopSitesTest::OnTopSitesAvailable));
+  EXPECT_EQ(2u, number_of_callbacks());
+  ASSERT_EQ(4u, urls().size());
+  EXPECT_EQ("http://bbc.com/", urls()[0].url.spec());
+  EXPECT_EQ(welcome_url(), urls()[1].url);
+  EXPECT_EQ(themes_url(), urls()[2].url);
+  EXPECT_EQ("http://google.com/", urls()[3].url.spec());
+
+  top_sites().RemovePinnedURL(GURL("http://google.com/"));
+  EXPECT_FALSE(top_sites().IsURLPinned(GURL("http://google.com/")));
+  top_sites().GetMostVisitedURLs(
+      &c,
+      NewCallback(static_cast<TopSitesTest*>(this),
+                  &TopSitesTest::OnTopSitesAvailable));
+
+  ASSERT_EQ(4u, urls().size());
+  EXPECT_EQ("http://bbc.com/", urls()[0].url.spec());
+  EXPECT_EQ("http://google.com/", urls()[1].url.spec());
+  EXPECT_EQ(welcome_url(), urls()[2].url);
+  EXPECT_EQ(themes_url(), urls()[3].url);
+}
+
+TEST_F(TopSitesTest, BlacklistingAndPinnedURLs) {
+  ChromeThread db_loop(ChromeThread::DB, MessageLoop::current());
+  MostVisitedURLList pages;
+  CancelableRequestConsumer c;
+  top_sites().GetMostVisitedURLs(
+      &c,
+      NewCallback(static_cast<TopSitesTest*>(this),
+                  &TopSitesTest::OnTopSitesAvailable));
+  top_sites().OnTopSitesAvailable(0, pages);
+  MessageLoop::current()->RunAllPending();
+
+  ASSERT_EQ(2u, urls().size());
+  EXPECT_EQ(welcome_url(), urls()[0].url);
+  EXPECT_EQ(themes_url(), urls()[1].url);
+
+  top_sites().AddPinnedURL(themes_url(), 1);
+  top_sites().AddBlacklistedURL(welcome_url());
+
+  top_sites().GetMostVisitedURLs(
+      &c,
+      NewCallback(static_cast<TopSitesTest*>(this),
+                  &TopSitesTest::OnTopSitesAvailable));
+
+  ASSERT_EQ(2u, urls().size());
+  EXPECT_EQ(GURL(), urls()[0].url);
+  EXPECT_EQ(themes_url(), urls()[1].url);
+
+}
+
+TEST_F(TopSitesTest, AddPrepopulatedPages) {
+  MostVisitedURLList pages;
+  top_sites().AddPrepopulatedPages(&pages);
+  ASSERT_EQ(2u, pages.size());
+  EXPECT_EQ(welcome_url(), pages[0].url);
+  EXPECT_EQ(themes_url(), pages[1].url);
+
+  pages.clear();
+
+  MostVisitedURL url = {themes_url()};
+  pages.push_back(url);
+
+  top_sites().AddPrepopulatedPages(&pages);
+
+  // Themes URL is already in pages; should not be added twice.
+  ASSERT_EQ(2u, pages.size());
+  EXPECT_EQ(themes_url(), pages[0].url);
+  EXPECT_EQ(welcome_url(), pages[1].url);
+}
+
+TEST_F(TopSitesTest, GetIndexForChromeStore) {
+  MostVisitedURLList pages;
+  EXPECT_EQ(0, top_sites().GetIndexForChromeStore(pages));
+
+  MostVisitedURL url = {themes_url()};
+  pages.push_back(url);
+  EXPECT_EQ(1, top_sites().GetIndexForChromeStore(pages));
+
+  // Store should be added in the first filler.
+  top_sites().AddPinnedURL(welcome_url(), 3);
+  top_sites().AddPinnedURL(GURL("http://google.com"), 5);
+  EXPECT_EQ(1, top_sites().GetIndexForChromeStore(pages));
+
+  GURL store_url = MostVisitedHandler::GetChromeStoreURLWithLocale();
+  url.url = store_url;
+  pages.push_back(url);
+
+  // Don't add store again.
+  EXPECT_EQ(-1, top_sites().GetIndexForChromeStore(pages));
+
+  pages.pop_back();
+  EXPECT_EQ(1, top_sites().GetIndexForChromeStore(pages));
+
+  pages.clear();
+  url.url = GURL("http://bbc.com/");
+  for (int i = 0; i < 8; i++) {
+    pages.push_back(url);
+  }
+
+  EXPECT_EQ(7, top_sites().GetIndexForChromeStore(pages));
+
+  pages[7].url = GURL("http://gmail.com");
+  top_sites().AddPinnedURL(GURL("http://gmail.com"), 7);
+
+  EXPECT_EQ(6, top_sites().GetIndexForChromeStore(pages));
+
+  // If it's blacklisted, it should not be added.
+  top_sites().AddBlacklistedURL(store_url);
+  EXPECT_EQ(-1, top_sites().GetIndexForChromeStore(pages));
+
+  top_sites().RemoveBlacklistedURL(store_url);
+  EXPECT_EQ(6, top_sites().GetIndexForChromeStore(pages));
+
+  top_sites().AddPinnedURL(GURL("http://bbc.com"), 2);
+  // All pinned - can't add store.
+  EXPECT_EQ(-1, top_sites().GetIndexForChromeStore(pages));
+}
+
 }  // namespace history