Implement zero suggest using psuggest.

Adds a new zero suggest experiment which displays results it gets from
psuggest (last 5 typed queries that led to result clicks).

These results also don't require sending url= parameter to the server.
Change the code so that url= gets sent if possible, but we still trigger
suggest on most visited and psuggest on any http/https pages since those
suggestions don't require sync.

Fixed an issue in autocomplete_controller where on query deletion in
zero suggest we would query all providers for extra results -> this
would return some junk suggestions on an empty input.

BUG=324043,343645

Review URL: https://codereview.chromium.org/189403006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@257721 0039d316-1c4b-4281-b951-d872f2087c98

12 files changed, 153 insertions, 62 deletions

diff --git a/chrome/app/generated_resources.grd b/chrome/app/generated_resources.grd
index 90c9d53..8f1789b 100644
--- a/chrome/app/generated_resources.grd
+++ b/chrome/app/generated_resources.grd
@@ -14741,6 +14741,9 @@ Do you accept?
     <message name="IDS_FLAGS_ZERO_SUGGEST_ETHER_NO_SERP" desc="A choice in dropdown dialog on about:flags page to show zero suggest on http pages only">
       Related URLs only
     </message>
+    <message name="IDS_FLAGS_ZERO_SUGGEST_PERSONALIZED" desc="A choice in dropdown dialog on about:flags page to show personalized zero-prefix suggestions">
+      Personalized
+    </message>
     <message name="IDS_FLAGS_DISABLE_IGNORE_AUTOCOMPLETE_OFF_NAME" desc="Name of the disable ignore autocomplete='off' lab">
       Disable ignore autocomplete='off'
     </message>
diff --git a/chrome/browser/about_flags.cc b/chrome/browser/about_flags.cc
index 16b8278..a818bb0 100644
--- a/chrome/browser/about_flags.cc
+++ b/chrome/browser/about_flags.cc
@@ -313,6 +313,8 @@ const Experiment::Choice kZeroSuggestExperimentsChoices[] = {
     switches::kEnableZeroSuggestEtherSerp, ""},
   { IDS_FLAGS_ZERO_SUGGEST_ETHER_NO_SERP,
     switches::kEnableZeroSuggestEtherNoSerp, ""},
+  { IDS_FLAGS_ZERO_SUGGEST_PERSONALIZED,
+    switches::kEnableZeroSuggestPersonalized, ""},
   { IDS_GENERIC_EXPERIMENT_CHOICE_DISABLED,
     switches::kDisableZeroSuggest, ""}
 };
diff --git a/chrome/browser/autocomplete/autocomplete_controller.cc b/chrome/browser/autocomplete/autocomplete_controller.cc
index eca78f6..8bb7e8b 100644
--- a/chrome/browser/autocomplete/autocomplete_controller.cc
+++ b/chrome/browser/autocomplete/autocomplete_controller.cc
@@ -161,7 +161,6 @@ AutocompleteController::AutocompleteController(
       stop_timer_duration_(OmniboxFieldTrial::StopTimerFieldTrialDuration()),
       done_(true),
       in_start_(false),
-      in_zero_suggest_(false),
       profile_(profile) {
   provider_types &= ~OmniboxFieldTrial::GetDisabledProviderTypes();
   if (provider_types & AutocompleteProvider::TYPE_BOOKMARK)
@@ -242,7 +241,6 @@ void AutocompleteController::Start(const AutocompleteInput& input) {
   stop_timer_.Stop();
 
   // Start the new query.
-  in_zero_suggest_ = false;
   in_start_ = true;
   base::TimeTicks start_time = base::TimeTicks::Now();
   for (ACProviders::iterator i(providers_.begin()); i != providers_.end();
@@ -315,7 +313,13 @@ void AutocompleteController::StartZeroSuggest(
     const base::string16& permanent_text) {
   if (zero_suggest_provider_ != NULL) {
     DCHECK(!in_start_);  // We should not be already running a query.
-    in_zero_suggest_ = true;
+
+    // Call Start() on all providers with INVALID input to clear out cached
+    // |matches_| to ensure they aren't used with zero suggest.
+    for (ACProviders::iterator i(providers_.begin()); i != providers_.end();
+        ++i)
+      (*i)->Start(AutocompleteInput(), false);
+
     zero_suggest_provider_->StartZeroSuggest(
         url, page_classification, permanent_text);
   }
@@ -356,21 +360,11 @@ void AutocompleteController::ExpireCopiedEntries() {
 }
 
 void AutocompleteController::OnProviderUpdate(bool updated_matches) {
-  if (in_zero_suggest_) {
-    // We got ZeroSuggest results before Start(). Show only those results,
-    // because results from other providers are stale.
-    result_.Reset();
-    result_.AppendMatches(zero_suggest_provider_->matches());
-    result_.SortAndCull(input_, profile_);
-    UpdateAssistedQueryStats(&result_);
-    NotifyChanged(true);
-  } else {
-    CheckIfDone();
-    // Multiple providers may provide synchronous results, so we only update the
-    // results if we're not in Start().
-    if (!in_start_ && (updated_matches || done_))
-      UpdateResult(false, false);
-  }
+  CheckIfDone();
+  // Multiple providers may provide synchronous results, so we only update the
+  // results if we're not in Start().
+  if (!in_start_ && (updated_matches || done_))
+    UpdateResult(false, false);
 }
 
 void AutocompleteController::AddProvidersInfo(
@@ -390,7 +384,6 @@ void AutocompleteController::ResetSession() {
   for (ACProviders::const_iterator i(providers_.begin()); i != providers_.end();
        ++i)
     (*i)->ResetSession();
-  in_zero_suggest_ = false;
 }
 
 void AutocompleteController::UpdateMatchDestinationURL(
@@ -439,8 +432,8 @@ void AutocompleteController::UpdateResult(
   AutocompleteResult last_result;
   last_result.Swap(&result_);
 
-  for (ACProviders::const_iterator i(providers_.begin()); i != providers_.end();
-       ++i)
+  for (ACProviders::const_iterator i(providers_.begin());
+       i != providers_.end(); ++i)
     result_.AppendMatches((*i)->matches());
 
   // Sort the matches and trim to a small number of "best" matches.
diff --git a/chrome/browser/autocomplete/autocomplete_controller.h b/chrome/browser/autocomplete/autocomplete_controller.h
index 26997b6..8931b96 100644
--- a/chrome/browser/autocomplete/autocomplete_controller.h
+++ b/chrome/browser/autocomplete/autocomplete_controller.h
@@ -235,9 +235,6 @@ class AutocompleteController : public AutocompleteProviderListener {
   // notifications until Start() has been invoked on all providers.
   bool in_start_;
 
-  // Has StartZeroSuggest() been called but not Start()?
-  bool in_zero_suggest_;
-
   Profile* profile_;
 
   DISALLOW_COPY_AND_ASSIGN(AutocompleteController);
diff --git a/chrome/browser/autocomplete/base_search_provider.cc b/chrome/browser/autocomplete/base_search_provider.cc
index c50ad2c..ce30af4 100644
--- a/chrome/browser/autocomplete/base_search_provider.cc
+++ b/chrome/browser/autocomplete/base_search_provider.cc
@@ -73,7 +73,6 @@ class SuggestionDeletionHandler : public net::URLFetcherDelegate {
   DISALLOW_COPY_AND_ASSIGN(SuggestionDeletionHandler);
 };
 
-
 SuggestionDeletionHandler::SuggestionDeletionHandler(
     const std::string& deletion_url,
     Profile* profile,
@@ -508,38 +507,28 @@ scoped_ptr<base::Value> BaseSearchProvider::DeserializeJsonData(
 }
 
 // static
-bool BaseSearchProvider::CanSendURL(
-    const GURL& current_page_url,
+bool BaseSearchProvider::ZeroSuggestEnabled(
     const GURL& suggest_url,
     const TemplateURL* template_url,
     AutocompleteInput::PageClassification page_classification,
     Profile* profile) {
-  if (!current_page_url.is_valid())
+  if (!OmniboxFieldTrial::InZeroSuggestFieldTrial())
+    return false;
+
+  // Make sure we are sending the suggest request through HTTPS to prevent
+  // exposing the current page URL or personalized results without encryption.
+  if (!suggest_url.SchemeIs(content::kHttpsScheme))
     return false;
 
-  // TODO(hfung): Show Most Visited on NTP with appropriate verbatim
-  // description when the user actively focuses on the omnibox as discussed in
-  // crbug/305366 if Most Visited (or something similar) will launch.
+  // Don't show zero suggest on the NTP.
+  // TODO(hfung): Experiment with showing MostVisited zero suggest on NTP
+  // under the conditions described in crbug.com/305366.
   if ((page_classification ==
        AutocompleteInput::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) ||
       (page_classification ==
        AutocompleteInput::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS))
     return false;
 
-  // Only allow HTTP URLs or HTTPS URLs for the same domain as the search
-  // provider.
-  if ((current_page_url.scheme() != content::kHttpScheme) &&
-      ((current_page_url.scheme() != content::kHttpsScheme) ||
-       !net::registry_controlled_domains::SameDomainOrHost(
-           current_page_url, suggest_url,
-           net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES)))
-    return false;
-
-  // Make sure we are sending the suggest request through HTTPS to prevent
-  // exposing the current page URL to networks before the search provider.
-  if (!suggest_url.SchemeIs(content::kHttpsScheme))
-    return false;
-
   // Don't run if there's no profile or in incognito mode.
   if (profile == NULL || profile->IsOffTheRecord())
     return false;
@@ -556,12 +545,37 @@ bool BaseSearchProvider::CanSendURL(
       SEARCH_ENGINE_GOOGLE)
     return false;
 
+  return true;
+}
+
+// static
+bool BaseSearchProvider::CanSendURL(
+    const GURL& current_page_url,
+    const GURL& suggest_url,
+    const TemplateURL* template_url,
+    AutocompleteInput::PageClassification page_classification,
+    Profile* profile) {
+  if (!ZeroSuggestEnabled(suggest_url, template_url, page_classification,
+                          profile))
+    return false;
+
+  if (!current_page_url.is_valid())
+    return false;
+
+  // Only allow HTTP URLs or HTTPS URLs for the same domain as the search
+  // provider.
+  if ((current_page_url.scheme() != content::kHttpScheme) &&
+      ((current_page_url.scheme() != content::kHttpsScheme) ||
+       !net::registry_controlled_domains::SameDomainOrHost(
+           current_page_url, suggest_url,
+           net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES)))
+    return false;
+
   // Check field trials and settings allow sending the URL on suggest requests.
   ProfileSyncService* service =
       ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile);
-  browser_sync::SyncPrefs sync_prefs(prefs);
-  if (!OmniboxFieldTrial::InZeroSuggestFieldTrial() ||
-      service == NULL ||
+  browser_sync::SyncPrefs sync_prefs(profile->GetPrefs());
+  if (service == NULL ||
       !service->IsSyncEnabledAndLoggedIn() ||
       !sync_prefs.GetPreferredDataTypes(syncer::UserTypes()).Has(
           syncer::PROXY_TABS) ||
diff --git a/chrome/browser/autocomplete/base_search_provider.h b/chrome/browser/autocomplete/base_search_provider.h
index d4b8963..d5bab9e 100644
--- a/chrome/browser/autocomplete/base_search_provider.h
+++ b/chrome/browser/autocomplete/base_search_provider.h
@@ -330,20 +330,31 @@ class BaseSearchProvider : public AutocompleteProvider,
   // otherwise.
   static scoped_ptr<base::Value> DeserializeJsonData(std::string json_data);
 
-  // Returns whether we can send the URL of the current page in any suggest
-  // requests.  Doing this requires that all the following hold:
+  // Returns whether the requirements for requesting zero suggest results
+  // are met. The requirements are
+  // * The user is enrolled in a zero suggest experiment.
+  // * The user is not on the NTP.
+  // * The suggest request is sent over HTTPS.  This avoids leaking the current
+  //   page URL or personal data in unencrypted network traffic.
   // * The user has suggest enabled in their settings and is not in incognito
   //   mode.  (Incognito disables suggest entirely.)
+  // * The user's suggest provider is Google.  We might want to allow other
+  //   providers to see this data someday, but for now this has only been
+  //   implemented for Google.
+  static bool ZeroSuggestEnabled(
+     const GURL& suggest_url,
+     const TemplateURL* template_url,
+     AutocompleteInput::PageClassification page_classification,
+     Profile* profile);
+
+  // Returns whether we can send the URL of the current page in any suggest
+  // requests.  Doing this requires that all the following hold:
+  // * ZeroSuggestEnabled() is true, so we meet the requirements above.
   // * The current URL is HTTP, or HTTPS with the same domain as the suggest
   //   server.  Non-HTTP[S] URLs (e.g. FTP/file URLs) may contain sensitive
   //   information.  HTTPS URLs may also contain sensitive information, but if
   //   they're on the same domain as the suggest server, then the relevant
   //   entity could have already seen/logged this data.
-  // * The suggest request is sent over HTTPS.  This avoids leaking the current
-  //   page URL in world-readable network traffic.
-  // * The user's suggest provider is Google.  We might want to allow other
-  //   providers to see this data someday, but for now this has only been
-  //   implemented for Google.  Also see next bullet.
   // * The user is OK in principle with sending URLs of current pages to their
   //   provider.  Today, there is no explicit setting that controls this, but if
   //   the user has tab sync enabled and tab sync is unencrypted, then they're
diff --git a/chrome/browser/autocomplete/zero_suggest_provider.cc b/chrome/browser/autocomplete/zero_suggest_provider.cc
index 1ea0c82..47251fb 100644
--- a/chrome/browser/autocomplete/zero_suggest_provider.cc
+++ b/chrome/browser/autocomplete/zero_suggest_provider.cc
@@ -90,6 +90,11 @@ void ZeroSuggestProvider::ResetSession() {
   // |field_trial_triggered_in_session_| unchanged and set
   // |field_trial_triggered_| to false since zero suggest is inactive now.
   field_trial_triggered_ = false;
+
+  // This call clears out |matches_| so that they don't pollute prefix-based
+  // queries.
+  // TODO(mariakhomenko): Change the model to clear |matches_| on Start() like
+  // all the other providers.
   Stop(true);
 }
 
@@ -130,16 +135,27 @@ void ZeroSuggestProvider::StartZeroSuggest(
      template_url_service_->GetDefaultSearchProvider();
   if (default_provider == NULL)
     return;
+
   base::string16 prefix;
   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
-  search_term_args.current_page_url = current_query_;
-  GURL suggest_url(default_provider->suggestions_url_ref().
-                   ReplaceSearchTerms(search_term_args));
-  if (!CanSendURL(current_page_url, suggest_url,
-          template_url_service_->GetDefaultSearchProvider(),
-          page_classification, profile_) ||
-      !OmniboxFieldTrial::InZeroSuggestFieldTrial())
+  GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
+      search_term_args));
+  if (!suggest_url.is_valid())
     return;
+
+  // No need to send the current page URL in personalized suggest field trial.
+  if (CanSendURL(current_page_url, suggest_url, default_provider,
+                 current_page_classification_, profile_) &&
+      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
+    // Update suggest_url to include the current_page_url.
+    search_term_args.current_page_url = current_query_;
+    suggest_url = GURL(default_provider->suggestions_url_ref().
+                       ReplaceSearchTerms(search_term_args));
+  } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url,
+                                                  current_page_url)) {
+    return;
+  }
+
   done_ = false;
   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
   // These may be useful on the NTP or more relevant to the user than server
@@ -364,3 +380,29 @@ int ZeroSuggestProvider::GetVerbatimRelevance() const {
   return results_.verbatim_relevance >= 0 ?
       results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
 }
+
+bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL(
+    const GURL& suggest_url,
+    const GURL& current_page_url) const {
+  if (!ZeroSuggestEnabled(suggest_url,
+                          template_url_service_->GetDefaultSearchProvider(),
+                          current_page_classification_, profile_))
+    return false;
+
+  // If we cannot send URLs, then only the MostVisited and Personalized
+  // variations can be shown.
+  if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
+      !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
+    return false;
+
+  // Only show zero suggest for HTTP[S] pages.
+  // TODO(mariakhomenko): We may be able to expand this set to include pages
+  // with other schemes (e.g. chrome://). That may require improvements to
+  // the formatting of the verbatim result returned by MatchForCurrentURL().
+  if (!current_page_url.is_valid() ||
+      ((current_page_url.scheme() != content::kHttpScheme) &&
+      (current_page_url.scheme() != content::kHttpsScheme)))
+    return false;
+
+  return true;
+}
diff --git a/chrome/browser/autocomplete/zero_suggest_provider.h b/chrome/browser/autocomplete/zero_suggest_provider.h
index 1a3a94a..8a8f6f7 100644
--- a/chrome/browser/autocomplete/zero_suggest_provider.h
+++ b/chrome/browser/autocomplete/zero_suggest_provider.h
@@ -111,6 +111,12 @@ class ZeroSuggestProvider : public BaseSearchProvider {
   // Returns the relevance score for the verbatim result.
   int GetVerbatimRelevance() const;
 
+  // Whether we can show zero suggest on |current_page_url| without
+  // sending |current_page_url| as a parameter to the server at |suggest_url|.
+  bool CanShowZeroSuggestWithoutSendingURL(
+      const GURL& suggest_url,
+      const GURL& current_page_url) const;
+
   // Used to build default search engine URLs for suggested queries.
   TemplateURLService* template_url_service_;
 
diff --git a/chrome/browser/omnibox/omnibox_field_trial.cc b/chrome/browser/omnibox/omnibox_field_trial.cc
index c7e3a1c..dce49e7 100644
--- a/chrome/browser/omnibox/omnibox_field_trial.cc
+++ b/chrome/browser/omnibox/omnibox_field_trial.cc
@@ -29,11 +29,16 @@ const char kHUPCullRedirectsFieldTrialName[] = "OmniboxHUPCullRedirects";
 const char kHUPCreateShorterMatchFieldTrialName[] =
     "OmniboxHUPCreateShorterMatch";
 const char kStopTimerFieldTrialName[] = "OmniboxStopTimer";
+
+// In dynamic field trials, we use these group names to switch between
+// different zero suggest implementations.
 const char kEnableZeroSuggestGroupPrefix[] = "EnableZeroSuggest";
 const char kEnableZeroSuggestMostVisitedGroupPrefix[] =
     "EnableZeroSuggestMostVisited";
 const char kEnableZeroSuggestAfterTypingGroupPrefix[] =
     "EnableZeroSuggestAfterTyping";
+const char kEnableZeroSuggestPersonalizedGroupPrefix[] =
+    "EnableZeroSuggestPersonalized";
 
 // The autocomplete dynamic field trial name prefix.  Each field trial is
 // configured dynamically and is retrieved automatically by Chrome during
@@ -291,6 +296,14 @@ bool OmniboxFieldTrial::InZeroSuggestAfterTypingFieldTrial() {
           kZeroSuggestVariantRule) == "AfterTyping";
 }
 
+bool OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() {
+  return HasDynamicFieldTrialGroupPrefix(
+      kEnableZeroSuggestPersonalizedGroupPrefix) ||
+      chrome_variations::GetVariationParamValue(
+          kBundledExperimentFieldTrialName,
+          kZeroSuggestVariantRule) == "Personalized";
+}
+
 bool OmniboxFieldTrial::ShortcutsScoringMaxRelevance(
     AutocompleteInput::PageClassification current_page_classification,
     int* max_relevance) {
diff --git a/chrome/browser/omnibox/omnibox_field_trial.h b/chrome/browser/omnibox/omnibox_field_trial.h
index b6d4155..e677fae 100644
--- a/chrome/browser/omnibox/omnibox_field_trial.h
+++ b/chrome/browser/omnibox/omnibox_field_trial.h
@@ -175,6 +175,10 @@ class OmniboxFieldTrial {
   // suggestions can continue to appear after the user has started typing.
   static bool InZeroSuggestAfterTypingFieldTrial();
 
+  // Returns whether the user is in a ZeroSuggest field trial, but should
+  // show recently searched-for queries instead.
+  static bool InZeroSuggestPersonalizedFieldTrial();
+
   // ---------------------------------------------------------
   // For the ShortcutsScoringMaxRelevance experiment that's part of the
   // bundled omnibox field trial.
diff --git a/chrome/common/chrome_switches.cc b/chrome/common/chrome_switches.cc
index a1ec5cc..bd8763b 100644
--- a/chrome/common/chrome_switches.cc
+++ b/chrome/common/chrome_switches.cc
@@ -1489,6 +1489,11 @@ const char kEnableZeroSuggestEtherNoSerp[] =
 const char kEnableZeroSuggestMostVisited[] =
     "enable-zero-suggest-most-visited";
 
+// Enables zero suggest functionality on Dev channel, showing recently typed
+// queries as default suggestions.
+const char kEnableZeroSuggestPersonalized[] =
+    "enable-zero-suggest-personalized";
+
 #endif
 
 #if defined(USE_ASH)
diff --git a/chrome/common/chrome_switches.h b/chrome/common/chrome_switches.h
index 2963c18..1781c7f 100644
--- a/chrome/common/chrome_switches.h
+++ b/chrome/common/chrome_switches.h
@@ -411,6 +411,7 @@ extern const char kEnableNewNTP[];
 extern const char kEnableZeroSuggestEtherSerp[];
 extern const char kEnableZeroSuggestEtherNoSerp[];
 extern const char kEnableZeroSuggestMostVisited[];
+extern const char kEnableZeroSuggestPersonalized[];
 #endif
 
 #if defined(USE_ASH)