First draft of web resource service; fetches data from a JSON feed

and stores it in user prefs, where it can be used by the new tab page.

BUG = http://crbug.com/13363

Review URL: http://codereview.chromium.org/125052

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@18766 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 370 insertions, 0 deletions

diff --git a/chrome/browser/web_resource/web_resource_service.cc b/chrome/browser/web_resource/web_resource_service.cc
new file mode 100644
index 0000000..7402025
--- /dev/null
+++ b/chrome/browser/web_resource/web_resource_service.cc
@@ -0,0 +1,268 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+#include "chrome/browser/web_resource/web_resource_service.h"
+
+#include "base/string_util.h"
+#include "base/time.h"
+#include "base/values.h"
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/chrome_thread.h"
+#include "chrome/browser/profile.h"
+#include "chrome/browser/net/url_fetcher.h"
+#include "chrome/common/pref_names.h"
+#include "googleurl/src/gurl.h"
+#include "net/base/load_flags.h"
+#include "net/url_request/url_request_status.h"
+
+const wchar_t* WebResourceService::kWebResourceTitle = L"title";
+const wchar_t* WebResourceService::kWebResourceThumb = L"thumbnail";
+const wchar_t* WebResourceService::kWebResourceSource = L"source";
+const wchar_t* WebResourceService::kWebResourceSnippet = L"snippet";
+const wchar_t* WebResourceService::kWebResourceURL = L"url";
+
+class WebResourceService::WebResourceFetcher
+    : public URLFetcher::Delegate {
+ public:
+  explicit WebResourceFetcher(WebResourceService* web_resource_service) :
+      ALLOW_THIS_IN_INITIALIZER_LIST(fetcher_factory_(this)),
+      web_resource_service_(web_resource_service) {
+  }
+
+  // Delay initial load of resource data into cache so as not to interfere
+  // with startup time.
+  void StartAfterDelay(int delay_ms) {
+    MessageLoop::current()->PostDelayedTask(FROM_HERE,
+      fetcher_factory_.NewRunnableMethod(&WebResourceFetcher::StartFetch),
+                                         delay_ms);
+  }
+
+  // Initializes the fetching of data from the resource server.  Data
+  // load calls OnURLFetchComplete.
+  void StartFetch() {
+    // First, put our next cache load on the MessageLoop.
+    MessageLoop::current()->PostDelayedTask(FROM_HERE,
+      fetcher_factory_.NewRunnableMethod(&WebResourceFetcher::StartFetch),
+                                         kCacheUpdateDelay);
+    // If we are still fetching data, exit.
+    if (web_resource_service_->in_fetch_)
+      return;
+
+    url_fetcher_.reset(new URLFetcher(GURL(
+        WideToUTF8(web_resource_service_->web_resource_server_)),
+        URLFetcher::GET, this));
+    // Do not let url fetcher affect existing state in profile (by setting
+    // cookies, for example.
+    url_fetcher_->set_load_flags(net::LOAD_DISABLE_CACHE |
+      net::LOAD_DO_NOT_SAVE_COOKIES);
+    url_fetcher_->set_request_context(Profile::GetDefaultRequestContext());
+    url_fetcher_->Start();
+  }
+
+  // From URLFetcher::Delegate.
+  void OnURLFetchComplete(const URLFetcher* source,
+                          const GURL& url,
+                          const URLRequestStatus& status,
+                          int response_code,
+                          const ResponseCookies& cookies,
+                          const std::string& data) {
+    // Delete the URLFetcher when this function exits.
+    scoped_ptr<URLFetcher> clean_up_fetcher(url_fetcher_.release());
+
+    // Don't parse data if attempt to download was unsuccessful.
+    // Stop loading new web resource data, and silently exit.
+    if (!status.is_success() || (response_code != 200))
+      return;
+
+    web_resource_service_->UpdateResourceCache(data);
+  }
+
+ private:
+  // So that we can delay our start so as not to affect start-up time; also,
+  // so that we can schedule future cache updates.
+  ScopedRunnableMethodFactory<WebResourceFetcher> fetcher_factory_;
+
+  // The tool that fetches the url data from the server.
+  scoped_ptr<URLFetcher> url_fetcher_;
+
+  // Our owner and creator.
+  scoped_ptr<WebResourceService> web_resource_service_;
+};
+
+// This class coordinates a web resource unpack and parse task which is run in
+// a separate process.  Results are sent back to this class and routed to
+// the WebResourceService.
+class WebResourceService::UnpackerClient
+    : public UtilityProcessHost::Client {
+ public:
+  UnpackerClient(WebResourceService* web_resource_service,
+                 const std::string& json_data)
+    : web_resource_service_(web_resource_service),
+      json_data_(json_data) {
+  }
+
+  void Start() {
+    AddRef();  // balanced in Cleanup.
+
+    if (web_resource_service_->resource_dispatcher_host_) {
+      ChromeThread::GetMessageLoop(ChromeThread::IO)->PostTask(FROM_HERE,
+          NewRunnableMethod(this, &UnpackerClient::StartProcessOnIOThread,
+                            web_resource_service_->resource_dispatcher_host_,
+                            MessageLoop::current()));
+    } else {
+      // TODO(mrc): unit tests here.
+    }
+  }
+
+ private:
+  virtual void OnUnpackWebResourceSucceeded(const ListValue& parsed_json) {
+    web_resource_service_->OnWebResourceUnpacked(parsed_json);
+    Release();
+  }
+
+  virtual void OnUnpackWebResourceFailed(const std::string& error_message) {
+    web_resource_service_->EndFetch();
+    Release();
+  }
+
+  void StartProcessOnIOThread(ResourceDispatcherHost* rdh,
+                              MessageLoop* file_loop) {
+    UtilityProcessHost* host = new UtilityProcessHost(rdh, this, file_loop);
+    // TODO(mrc): get proper file path when we start using web resources
+    // that need to be unpacked.
+    host->StartWebResourceUnpacker(json_data_);
+  }
+
+  scoped_refptr<WebResourceService> web_resource_service_;
+
+  // Holds raw JSON string.
+  const std::string& json_data_;
+};
+
+const wchar_t* WebResourceService::kDefaultResourceServer =
+//    L"http://hokiepokie.nyc.corp.google.com:8125/"
+//    L"labs/popgadget/world?view=json";
+    L"http://sites.google.com/a/chromium.org/dev/developers/"
+    L"design-documents/web_resources/popgadget_test.json";
+
+const char* WebResourceService::kResourceDirectoryName =
+    "Resources";
+
+WebResourceService::WebResourceService(Profile* profile,
+                                       MessageLoop* backend_loop) :
+    prefs_(profile->GetPrefs()),
+    web_resource_dir_(profile->GetPath().AppendASCII(kResourceDirectoryName)),
+    backend_loop_(backend_loop),
+    in_fetch_(false) {
+  Init();
+}
+
+WebResourceService::~WebResourceService() { }
+
+void WebResourceService::Init() {
+  resource_dispatcher_host_ = g_browser_process->resource_dispatcher_host();
+  web_resource_fetcher_ = new WebResourceFetcher(this);
+  prefs_->RegisterStringPref(prefs::kNTPWebResourceCacheUpdate, L"0");
+  // TODO(mrc): make sure server name is valid.
+  web_resource_server_ = prefs_->HasPrefPath(prefs::kNTPWebResourceServer) ?
+      prefs_->GetString(prefs::kNTPWebResourceServer) :
+      kDefaultResourceServer;
+}
+
+void WebResourceService::EndFetch() {
+  in_fetch_ = false;
+}
+
+void WebResourceService::OnWebResourceUnpacked(const ListValue& parsed_json) {
+  // Get dictionary of cached preferences.
+  web_resource_cache_ =
+      prefs_->GetMutableDictionary(prefs::kNTPWebResourceCache);
+  ListValue::const_iterator wr_iter = parsed_json.begin();
+  int wr_counter = 0;
+
+  // These values store the data for each new web resource item.
+  std::wstring result_snippet;
+  std::wstring result_url;
+  std::wstring result_source;
+  std::wstring result_title;
+  std::wstring result_title_type;
+  std::wstring result_thumbnail;
+
+  // Iterate through newly parsed preferences, replacing stale cache with
+  // new data.
+  // TODO(mrc): make this smarter, so it actually only replaces stale data,
+  // instead of overwriting the whole thing every time.
+  while (wr_iter != parsed_json.end() &&
+         wr_counter < kMaxResourceCacheSize) {
+    // Each item is stored in the form of a dictionary.
+    // See web_resource_handler.h for format (this will change until
+    // web resource services are solidified!).
+    if (!(*wr_iter)->IsType(Value::TYPE_DICTIONARY))
+      continue;
+    DictionaryValue* wr_dict =
+        static_cast<DictionaryValue*>(*wr_iter);
+
+    // Get next space for resource in prefs file.
+    Value* current_wr;
+    std::wstring wr_counter_str = IntToWString(wr_counter);
+    // Create space if it doesn't exist yet.
+    if (!web_resource_cache_->Get(wr_counter_str, &current_wr) ||
+      !current_wr->IsType(Value::TYPE_DICTIONARY)) {
+        current_wr = new DictionaryValue();
+        web_resource_cache_->Set(wr_counter_str, current_wr);
+    }
+    DictionaryValue* wr_cache_dict =
+      static_cast<DictionaryValue*>(current_wr);
+
+    // Update the resource cache.
+    wr_cache_dict->SetString(L"index", wr_counter_str);
+
+    if (wr_dict->GetString(kWebResourceSnippet, &result_snippet))
+      wr_cache_dict->SetString(kWebResourceSnippet, result_snippet);
+    if (wr_dict->GetString(kWebResourceSource, &result_source))
+      wr_cache_dict->SetString(kWebResourceSource, result_source);
+    if (wr_dict->GetString(kWebResourceURL, &result_url))
+      wr_cache_dict->SetString(kWebResourceURL, result_url);
+    if (wr_dict->GetString(kWebResourceTitle, &result_title))
+      wr_cache_dict->SetString(kWebResourceTitle, result_title);
+    if (wr_dict->GetString(kWebResourceThumb, &result_thumbnail))
+      wr_cache_dict->SetString(kWebResourceThumb, result_thumbnail);
+
+    wr_counter++;
+    wr_iter++;
+  }
+  EndFetch();
+}
+
+void WebResourceService::StartAfterDelay() {
+  int64 delay = kStartResourceFetchDelay;
+  // Check whether we have ever put a value in the web resource cache;
+  // if so, pull it out and see if it's time to update again.
+  if (prefs_->HasPrefPath(prefs::kNTPWebResourceCacheUpdate)) {
+    std::wstring last_update_pref =
+      prefs_->GetString(prefs::kNTPWebResourceCacheUpdate);
+    int64 ms_since_update =
+        (base::Time::Now() - base::Time::FromDoubleT(
+        StringToDouble(WideToASCII(last_update_pref)))).InMilliseconds();
+
+    delay = kStartResourceFetchDelay +
+        (ms_since_update > kCacheUpdateDelay ?
+         0 : kCacheUpdateDelay - ms_since_update);
+  }
+
+  // Start fetch and wait for UpdateResourceCache.
+  DCHECK(delay >= kStartResourceFetchDelay &&
+         delay <= kCacheUpdateDelay);
+  web_resource_fetcher_->StartAfterDelay(static_cast<int>(delay));
+}
+
+void WebResourceService::UpdateResourceCache(const std::string& json_data) {
+  UnpackerClient* client = new UnpackerClient(this, json_data);
+  client->Start();
+
+  // Update resource server and cache update time in preferences.
+  prefs_->SetString(prefs::kNTPWebResourceCacheUpdate,
+      DoubleToWString(base::Time::Now().ToDoubleT()));
+  prefs_->SetString(prefs::kNTPWebResourceServer, web_resource_server_);
+}
+
diff --git a/chrome/browser/web_resource/web_resource_service.h b/chrome/browser/web_resource/web_resource_service.h
new file mode 100644
index 0000000..7e50ee5
--- /dev/null
+++ b/chrome/browser/web_resource/web_resource_service.h
@@ -0,0 +1,102 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_WEB_RESOURCE_WEB_RESOURCE_SERVICE_H_
+#define CHROME_BROWSER_WEB_RESOURCE_WEB_RESOURCE_SERVICE_H_
+
+#include <string>
+
+#include "chrome/browser/utility_process_host.h"
+#include "chrome/common/pref_service.h"
+#include "chrome/common/web_resource/web_resource_unpacker.h"
+
+class Profile;
+
+class WebResourceService
+    : public UtilityProcessHost::Client {
+ public:
+  WebResourceService(Profile* profile,
+                     MessageLoop* backend_loop);
+  ~WebResourceService();
+
+  // Sleep until cache needs to be updated, but always for at least 5 seconds
+  // so we don't interfere with startup.  Then begin updating resources.
+  void StartAfterDelay();
+
+  // We have successfully pulled data from a resource server; now launch
+  // the process that will parse the JSON, and then update the cache.
+  void UpdateResourceCache(const std::string& json_data);
+
+  // Right now, these values correspond to data pulled from the popgadget
+  // JSON feed.  Once we have decided on the final format for the
+  // web resources servers, these will probably change.
+  static const wchar_t* kWebResourceTitle;
+  static const wchar_t* kWebResourceThumb;
+  static const wchar_t* kWebResourceSource;
+  static const wchar_t* kWebResourceSnippet;
+  static const wchar_t* kWebResourceURL;
+
+  // Default server from which to gather resources.
+  // For now, hard-coded to test JSON data hosted on chromium.org.
+  // Starting 6/22, poptart server will be ready to host data.
+  // Future: more servers and different kinds of data will be served.
+  static const wchar_t* kDefaultResourceServer;
+
+ private:
+  class WebResourceFetcher;
+  friend class WebResourceFetcher;
+
+  class UnpackerClient;
+
+  void Init();
+
+  // Set in_fetch_ to false, clean up temp directories (in the future).
+  void EndFetch();
+
+  // Puts parsed json data in the right places, and writes to prefs file.
+  void OnWebResourceUnpacked(const ListValue& parsed_json);
+
+  // We need to be able to load parsed resource data into preferences file,
+  // and get proper install directory.
+  PrefService* prefs_;
+
+  FilePath web_resource_dir_;
+
+  // Server from which we are currently pulling web resource data.
+  std::wstring web_resource_server_;
+
+  // Whenever we update resource cache, schedule another task.
+  MessageLoop* backend_loop_;
+
+  WebResourceFetcher* web_resource_fetcher_;
+
+  ResourceDispatcherHost* resource_dispatcher_host_;
+
+  // Gets mutable dictionary attached to user's preferences, so that we
+  // can write resource data back to user's pref file.
+  DictionaryValue* web_resource_cache_;
+
+  // True if we are currently mid-fetch.  If we are asked to start a fetch
+  // when we are still fetching resource data, schedule another one in
+  // kCacheUpdateDelay time, and silently exit.
+  bool in_fetch_;
+
+  // Maximum number of cached resources available.
+  static const int kMaxResourceCacheSize = 3;
+
+  // Delay on first fetch so we don't interfere with startup.
+  static const int kStartResourceFetchDelay = 5000;
+
+  // Delay between calls to update the cache (4 hours).
+  static const int kCacheUpdateDelay = 4 * 60 * 60 * 1000;
+
+  // Name of directory inside the profile where we will store resource-related
+  // data (for now, thumbnail images).
+  static const char* kResourceDirectoryName;
+
+  DISALLOW_COPY_AND_ASSIGN(WebResourceService);
+};
+
+#endif  // CHROME_BROWSER_WEB_RESOURCE_WEB_RESOURCE_SERVICE_H_
+