Integration with Toolbar autofill servers.

Still to do:
  1. Update upload rates based on response
  2. Differentiate autofilled/not uploads
  3. Do not query if there is no items in autofill
  4. Fix forms to correspond to the data queried by toolbar
BUG=none
TEST=Should request data from toolbar servers
Review URL: http://codereview.chromium.org/1119004

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@42260 0039d316-1c4b-4281-b951-d872f2087c98

7 files changed, 330 insertions, 25 deletions

diff --git a/chrome/browser/autofill/autofill_download.cc b/chrome/browser/autofill/autofill_download.cc
new file mode 100644
index 0000000..0d650d1
--- /dev/null
+++ b/chrome/browser/autofill/autofill_download.cc
@@ -0,0 +1,126 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/autofill/autofill_download.h"
+
+#include "base/logging.h"
+#include "base/rand_util.h"
+#include "base/stl_util-inl.h"
+#include "chrome/browser/profile.h"
+
+#if defined(GOOGLE_CHROME_BUILD)
+#include "chrome/browser/autofill/internal/autofill_download_internal.h"
+#else
+#define AUTO_FILL_QUERY_SERVER_REQUEST_URL "http://disabled"
+#define AUTO_FILL_UPLOAD_SERVER_REQUEST_URL "http://disabled"
+#endif
+
+namespace {
+// We only send a fraction of the forms to upload server.
+// The rate for positive/negative matches potentially could be different.
+const double kAutoFillPositiveUploadRate = 0.01;
+const double kAutoFillNegativeUploadRate = 0.01;
+}  // namespace
+
+AutoFillDownloadManager::AutoFillDownloadManager()
+  : observer_(NULL) {
+}
+
+AutoFillDownloadManager::~AutoFillDownloadManager() {
+  STLDeleteContainerPairFirstPointers(url_fetchers_.begin(),
+                                      url_fetchers_.end());
+}
+
+void AutoFillDownloadManager::SetObserver(
+    AutoFillDownloadManager::Observer *observer) {
+  if (observer) {
+    DCHECK(!observer_);
+    observer_ = observer;
+  } else {
+    observer_ = NULL;
+  }
+}
+
+bool AutoFillDownloadManager::StartRequest(const std::string& form_xml,
+                                           const std::string& form_signature,
+                                           bool query_data,
+                                           bool form_was_matched) {
+  if (!query_data) {
+    // Check if we need to upload form.
+    double upload_rate = form_was_matched ? kAutoFillPositiveUploadRate :
+      kAutoFillNegativeUploadRate;
+    if (base::RandDouble() > upload_rate) {
+      LOG(INFO) << "AutoFillDownloadManager: Upload request is ignored";
+      if (observer_)
+        observer_->OnUploadedAutoFillHeuristics(form_signature);
+      return true;
+    }
+  }
+
+  std::string request_url;
+  if (query_data)
+    request_url = AUTO_FILL_QUERY_SERVER_REQUEST_URL;
+  else
+    request_url = AUTO_FILL_UPLOAD_SERVER_REQUEST_URL;
+
+  URLFetcher *fetcher =
+      URLFetcher::Create(0, GURL(request_url), URLFetcher::POST, this);
+  FormRequestData data;
+  data.form_signature = form_signature;
+  data.query = query_data;
+  url_fetchers_[fetcher] = data;
+  fetcher->set_request_context(Profile::GetDefaultRequestContext());
+  fetcher->set_upload_data("text/plain", form_xml);
+  fetcher->Start();
+  return true;
+}
+
+bool AutoFillDownloadManager::CancelRequest(const std::string& form_signature,
+                                            bool query_data) {
+  for (std::map<URLFetcher *, FormRequestData>::iterator it =
+       url_fetchers_.begin();
+       it != url_fetchers_.end();
+       ++it) {
+    if (it->second.form_signature == form_signature &&
+        it->second.query == query_data) {
+      delete it->first;
+      url_fetchers_.erase(it);
+      return true;
+    }
+  }
+  return false;
+}
+
+void AutoFillDownloadManager::OnURLFetchComplete(const URLFetcher* source,
+                                                 const GURL& url,
+                                                 const URLRequestStatus& status,
+                                                 int response_code,
+                                                 const ResponseCookies& cookies,
+                                                 const std::string& data) {
+  std::map<URLFetcher *, FormRequestData>::iterator it =
+      url_fetchers_.find(const_cast<URLFetcher*>(source));
+  DCHECK(it != url_fetchers_.end());
+  std::string type_of_request(it->second.query ? "query" : "upload");
+  const int kHttpResponseOk = 200;
+  if (response_code != kHttpResponseOk) {
+    LOG(INFO) << "AutoFillDownloadManager: " << type_of_request <<
+        " request has failed with response" << response_code;
+    if (observer_) {
+      observer_->OnHeuristicsRequestError(it->second.form_signature,
+                                          response_code);
+    }
+  } else {
+    LOG(INFO) << "AutoFillDownloadManager: " << type_of_request <<
+        " request has succeeded";
+    if (it->second.query) {
+      if (observer_)
+        observer_->OnLoadedAutoFillHeuristics(it->second.form_signature, data);
+    } else {
+      if (observer_)
+        observer_->OnUploadedAutoFillHeuristics(it->second.form_signature);
+    }
+  }
+}
+
+
diff --git a/chrome/browser/autofill/autofill_download.h b/chrome/browser/autofill/autofill_download.h
new file mode 100644
index 0000000..c9e6ed2
--- /dev/null
+++ b/chrome/browser/autofill/autofill_download.h
@@ -0,0 +1,88 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_AUTOFILL_AUTOFILL_DOWNLOAD_H_
+#define CHROME_BROWSER_AUTOFILL_AUTOFILL_DOWNLOAD_H_
+
+#include <map>
+#include <string>
+
+#include "base/scoped_ptr.h"
+#include "base/scoped_vector.h"
+#include "base/string16.h"
+#include "chrome/browser/autofill/autofill_profile.h"
+#include "chrome/browser/autofill/field_types.h"
+#include "chrome/browser/autofill/form_structure.h"
+#include "chrome/browser/net/url_fetcher.h"
+
+// Handles getting and updating AutoFill heuristics.
+class AutoFillDownloadManager : public URLFetcher::Delegate {
+ public:
+  // An interface used to notify clients of AutoFillDownloadManager.
+  class Observer {
+   public:
+    // Called when heuristic successfully received from server.
+    // |form_signature| - the signature of the requesting form.
+    // |heuristic_xml| - server response.
+    virtual void OnLoadedAutoFillHeuristics(
+        const std::string& form_signature,
+        const std::string& heuristic_xml) = 0;
+    // Called when heuristic either successfully considered for upload and
+    // not send or uploaded.
+    // |form_signature| - the signature of the requesting form.
+    virtual void OnUploadedAutoFillHeuristics(
+        const std::string& form_signature) = 0;
+    // Called when there was an error during the request.
+    // |form_signature| - the signature of the requesting form.
+    // |http_error| - http error code.
+    virtual void OnHeuristicsRequestError(const std::string& form_signature,
+                                          int http_error) = 0;
+   protected:
+    virtual ~Observer() {}
+  };
+
+  AutoFillDownloadManager();
+  virtual ~AutoFillDownloadManager();
+
+  // |observer| - observer to notify on successful completion or error.
+  void SetObserver(AutoFillDownloadManager::Observer *observer);
+
+  // Initiates request to AutoFill servers to download/upload heuristics
+  // |form_xml| - form structure XML to upload/download.
+  // |form_signature| - form signature hash.
+  // |query_data| - if true the data is queried and observer notified with new
+  // data, if available. If false heuristic data is uploaded to our servers.
+  // |form_was_matched| - if |query_data| is false indicates if the form was
+  // matched. Ignored otherwise.
+  bool StartRequest(const std::string& form_xml,
+                    const std::string& form_signature,
+                    bool query_data,
+                    bool form_was_matched);
+
+  bool CancelRequest(const std::string& form_signature, bool query_data);
+
+ protected:
+  // URLFetcher::Delegate implementation:
+  virtual void OnURLFetchComplete(const URLFetcher* source,
+                                  const GURL& url,
+                                  const URLRequestStatus& status,
+                                  int response_code,
+                                  const ResponseCookies& cookies,
+                                  const std::string& data);
+
+ private:
+  struct FormRequestData {
+    std::string form_signature;
+    bool query;
+  };
+
+  // For each requested form for both query and upload we create a separate
+  // request and save its info. As url fetcher is identified by its address
+  // we use a map between fetchers and info.
+  std::map<URLFetcher *, FormRequestData> url_fetchers_;
+  AutoFillDownloadManager::Observer *observer_;
+};
+
+#endif  // CHROME_BROWSER_AUTOFILL_AUTOFILL_DOWNLOAD_H_
+
diff --git a/chrome/browser/autofill/autofill_manager.cc b/chrome/browser/autofill/autofill_manager.cc
index 0f385de..4ffdc20 100644
--- a/chrome/browser/autofill/autofill_manager.cc
+++ b/chrome/browser/autofill/autofill_manager.cc
@@ -9,6 +9,7 @@
 #include "base/command_line.h"
 #include "chrome/browser/autofill/autofill_dialog.h"
 #include "chrome/browser/autofill/autofill_infobar_delegate.h"
+#include "chrome/browser/autofill/autofill_xml_parser.h"
 #include "chrome/browser/autofill/form_structure.h"
 #include "chrome/browser/pref_service.h"
 #include "chrome/browser/profile.h"
@@ -29,12 +30,14 @@ AutoFillManager::AutoFillManager(TabContents* tab_contents)
   personal_data_ =
       tab_contents_->profile()->GetOriginalProfile()->GetPersonalDataManager();
   DCHECK(personal_data_);
+  download_manager_.SetObserver(this);
 }
 
 AutoFillManager::~AutoFillManager() {
   // This is NULL in the MockAutoFillManager.
   if (personal_data_)
     personal_data_->RemoveObserver(this);
+  download_manager_.SetObserver(NULL);
 }
 
 // static
@@ -87,6 +90,14 @@ void AutoFillManager::FormsSeen(
     FormStructure* form_structure = new FormStructure(*iter);
     DeterminePossibleFieldTypes(form_structure);
     form_structures_.push_back(form_structure);
+    std::string request_xml;
+    if (form_structure->IsAutoFillable() &&
+        form_structure->EncodeUploadRequest(true, true, &request_xml)) {
+      download_manager_.StartRequest(request_xml,
+                                     form_structure->FormSignature(),
+                                     true,
+                                     false);
+    }
   }
 }
 
@@ -282,6 +293,48 @@ void AutoFillManager::Reset() {
   form_structures_.reset();
 }
 
+void AutoFillManager::OnLoadedAutoFillHeuristics(
+    const std::string& form_signature,
+    const std::string& heuristic_xml) {
+  for (ScopedVector<FormStructure>::iterator it = form_structures_.begin();
+      it != form_structures_.end();
+      ++it) {
+    if ((*it)->FormSignature() == form_signature) {
+      // Create a vector of AutoFillFieldTypes,
+      // to assign the parsed field types to.
+      std::vector<AutoFillFieldType> field_types;
+      UploadRequired upload_required = USE_UPLOAD_RATES;
+
+      // Create a parser.
+      AutoFillQueryXmlParser parse_handler(&field_types, &upload_required);
+      buzz::XmlParser parser(&parse_handler);
+      parser.Parse(heuristic_xml.c_str(), heuristic_xml.length(), true);
+      if (parse_handler.succeeded()) {
+        DCHECK(field_types.size() == (*it)->field_count());
+        if (field_types.size() == (*it)->field_count()) {
+          for (size_t i = 0; i < (*it)->field_count(); ++i) {
+            if (field_types[i] != NO_SERVER_DATA &&
+                field_types[i] != UNKNOWN_TYPE) {
+             FieldTypeSet types = (*it)->field(i)->possible_types();
+             types.insert(field_types[i]);
+             (*it)->set_possible_types(i, types);
+            }
+          }
+        }
+        return;
+      }
+    }
+  }
+}
+
+void AutoFillManager::OnUploadedAutoFillHeuristics(
+    const std::string& form_signature) {
+}
+
+void AutoFillManager::OnHeuristicsRequestError(
+    const std::string& form_signature, int http_error) {
+}
+
 void AutoFillManager::DeterminePossibleFieldTypes(
     FormStructure* form_structure) {
   // TODO(jhawkins): Update field text.
@@ -311,10 +364,15 @@ void AutoFillManager::HandleSubmit() {
 
 void AutoFillManager::UploadFormData() {
   std::string xml;
-  bool ok = upload_form_structure_->EncodeUploadRequest(false, &xml);
+  bool ok = upload_form_structure_->EncodeUploadRequest(false, false, &xml);
   DCHECK(ok);
 
-  // TODO(jhawkins): Initiate the upload request thread.
+  // TODO(georgey): enable upload request when we make sure that our data is in
+  // line with toolbar data:
+  // download_manager_.StartRequest(xml,
+  //                                upload_form_structure_->FormSignature(),
+  //                                false,
+  //                                form_is_autofilled);
 }
 
 bool AutoFillManager::IsAutoFillEnabled() {
diff --git a/chrome/browser/autofill/autofill_manager.h b/chrome/browser/autofill/autofill_manager.h
index e09cecf..f56a5f2 100644
--- a/chrome/browser/autofill/autofill_manager.h
+++ b/chrome/browser/autofill/autofill_manager.h
@@ -6,10 +6,12 @@
 #define CHROME_BROWSER_AUTOFILL_AUTOFILL_MANAGER_H_
 
 #include <vector>
+#include <string>
 
 #include "base/scoped_ptr.h"
 #include "base/scoped_vector.h"
 #include "chrome/browser/autofill/autofill_dialog.h"
+#include "chrome/browser/autofill/autofill_download.h"
 #include "chrome/browser/autofill/personal_data_manager.h"
 #include "chrome/browser/renderer_host/render_view_host_delegate.h"
 
@@ -30,7 +32,8 @@ class TabContents;
 // forms.
 class AutoFillManager : public RenderViewHostDelegate::AutoFill,
                         public AutoFillDialogObserver,
-                        public PersonalDataManager::Observer {
+                        public PersonalDataManager::Observer,
+                        public AutoFillDownloadManager::Observer {
  public:
   explicit AutoFillManager(TabContents* tab_contents);
   virtual ~AutoFillManager();
@@ -73,6 +76,13 @@ class AutoFillManager : public RenderViewHostDelegate::AutoFill,
   // Resets the stored form data.
   virtual void Reset();
 
+  // AutoFillDownloadManager::Observer implementation:
+  virtual void OnLoadedAutoFillHeuristics(const std::string& form_signature,
+                                          const std::string& heuristic_xml);
+  virtual void OnUploadedAutoFillHeuristics(const std::string& form_signature);
+  virtual void OnHeuristicsRequestError(const std::string& form_signature,
+                                        int http_error);
+
   // Uses heuristics and existing personal data to determine the possible field
   // types.
   void DeterminePossibleFieldTypes(FormStructure* form_structure);
@@ -103,6 +113,9 @@ class AutoFillManager : public RenderViewHostDelegate::AutoFill,
   // May be NULL.  NULL indicates OTR.
   PersonalDataManager* personal_data_;
 
+  // Handles queries and uploads to AutoFill servers.
+  AutoFillDownloadManager download_manager_;
+
   // Our copy of the form data.
   ScopedVector<FormStructure> form_structures_;
 
diff --git a/chrome/browser/autofill/form_structure.cc b/chrome/browser/autofill/form_structure.cc
index 4841da8..dd9cfaa 100644
--- a/chrome/browser/autofill/form_structure.cc
+++ b/chrome/browser/autofill/form_structure.cc
@@ -90,53 +90,69 @@ FormStructure::FormStructure(const FormFieldValues& values)
 }
 
 bool FormStructure::EncodeUploadRequest(bool auto_fill_used,
+                                        bool query,
                                         std::string* encoded_xml) const {
   bool auto_fillable = IsAutoFillable();
   DCHECK(auto_fillable);  // Caller should've checked for search pages.
   if (!auto_fillable)
     return false;
 
-  buzz::XmlElement autofill_upload(buzz::QName("autofillupload"));
+  buzz::XmlElement autofil_request_xml(query ? buzz::QName("autofillquery") :
+                                               buzz::QName("autofillupload"));
+  buzz::XmlElement *encompassing_xml_element = &autofil_request_xml;
+  if (query)
+    encompassing_xml_element = new buzz::XmlElement(buzz::QName("form"));
 
-  // Attributes for the <autofillupload> element.
+  // Attributes for the <autofillupload>/<autofillquery> element.
   //
   // TODO(jhawkins): Work with toolbar devs to make a spec for autofill clients.
   // For now these values are hacked from the toolbar code.
-  autofill_upload.SetAttr(buzz::QName(kAttributeClientVersion),
-                          "6.1.1715.1442/en (GGLL)");
+  autofil_request_xml.SetAttr(buzz::QName(kAttributeClientVersion),
+                              "6.1.1715.1442/en (GGLL)");
 
-  autofill_upload.SetAttr(buzz::QName(kAttributeFormSignature),
-                          FormSignature());
+  encompassing_xml_element->SetAttr(query ? buzz::QName(kAttributeSignature) :
+                                    buzz::QName(kAttributeFormSignature),
+                                    FormSignature());
 
-  autofill_upload.SetAttr(buzz::QName(kAttributeAutoFillUsed),
-                          auto_fill_used ? "true" : "false");
+  if (!query) {
+    autofil_request_xml.SetAttr(buzz::QName(kAttributeAutoFillUsed),
+                                auto_fill_used ? "true" : "false");
 
-  // TODO(jhawkins): Hook this up to the personal data manager.
-  // personaldata_manager_->GetDataPresent();
-  autofill_upload.SetAttr(buzz::QName(kAttributeDataPresent), "");
+    // TODO(jhawkins): Hook this up to the personal data manager.
+    // personaldata_manager_->GetDataPresent();
+    autofil_request_xml.SetAttr(buzz::QName(kAttributeDataPresent), "");
+  }
 
   // Add the child nodes for the form fields.
   for (size_t index = 0; index < field_count(); index++) {
     const AutoFillField* field = fields_[index];
+    if (!query) {
     FieldTypeSet types = field->possible_types();
-    for (FieldTypeSet::const_iterator type = types.begin();
-         type != types.end(); type++) {
+      for (FieldTypeSet::const_iterator type = types.begin();
+           type != types.end(); type++) {
+        buzz::XmlElement *field_element = new buzz::XmlElement(
+            buzz::QName(kXMLElementField));
+
+        field_element->SetAttr(buzz::QName(kAttributeSignature),
+                               field->FieldSignature());
+        field_element->SetAttr(buzz::QName(kAttributeAutoFillType),
+                               IntToString(*type));
+        encompassing_xml_element->AddElement(field_element);
+      }
+    } else {
       buzz::XmlElement *field_element = new buzz::XmlElement(
           buzz::QName(kXMLElementField));
-
       field_element->SetAttr(buzz::QName(kAttributeSignature),
                              field->FieldSignature());
-
-      field_element->SetAttr(buzz::QName(kAttributeAutoFillType),
-                             IntToString(*type));
-
-      autofill_upload.AddElement(field_element);
+      encompassing_xml_element->AddElement(field_element);
     }
   }
+  if (query)
+    autofil_request_xml.AddElement(encompassing_xml_element);
 
   // Obtain the XML structure as a string.
   *encoded_xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
-  *encoded_xml += autofill_upload.Str().c_str();
+  *encoded_xml += autofil_request_xml.Str().c_str();
 
   return true;
 }
diff --git a/chrome/browser/autofill/form_structure.h b/chrome/browser/autofill/form_structure.h
index 4376987..3802aed 100644
--- a/chrome/browser/autofill/form_structure.h
+++ b/chrome/browser/autofill/form_structure.h
@@ -36,8 +36,10 @@ class FormStructure {
  public:
   explicit FormStructure(const webkit_glue::FormFieldValues& values);
 
-  // Encodes the XML upload request from this FormStructure.
-  bool EncodeUploadRequest(bool auto_fill_used, std::string* encoded_xml) const;
+  // Encodes the XML query or upload request from this FormStructure.
+  // |query| - true means request is a query, upload otherwise.
+  bool EncodeUploadRequest(bool auto_fill_used, bool query,
+                           std::string* encoded_xml) const;
 
   // Runs several heuristics against the form fields to determine their possible
   // types.
diff --git a/chrome/chrome_browser.gypi b/chrome/chrome_browser.gypi
index 2563588..bb18a87 100755
--- a/chrome/chrome_browser.gypi
+++ b/chrome/chrome_browser.gypi
@@ -121,6 +121,8 @@
         # Windows version of autofill dialog is defined in
         # 'browser/views/autofill_profiles_view_win.cc'
         'browser/autofill/autofill_dialog.h',
+        'browser/autofill/autofill_download.cc',
+        'browser/autofill/autofill_download.h',
         'browser/autofill/autofill_field.cc',
         'browser/autofill/autofill_field.h',
         'browser/autofill/autofill_infobar_delegate.cc',