Experimental integration of delta compression content encoding

The command line option "sdch-enable" enables support of sdch
and automtic lazy download of dictionaries.  Optionally it
can select a singular domain to work from.

By default, all domains are enabled when the flag is used.
"-sdch-enable=".google.com"  Enables it only for Google.

When the switch is not set on the command line, all this
code is completely disabled.

Still TBD:
a) Finish implementation of security details (much of it is in place)
b) Add tests for security details.


r=huanr,ajenjo,kmixter

Review URL: http://codereview.chromium.org/461

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@2443 0039d316-1c4b-4281-b951-d872f2087c98

8 files changed, 170 insertions, 0 deletions

diff --git a/chrome/browser/browser.vcproj b/chrome/browser/browser.vcproj
index fc3c31b..e87a3e0 100644
--- a/chrome/browser/browser.vcproj
+++ b/chrome/browser/browser.vcproj
@@ -1861,6 +1861,14 @@
 				RelativePath=".\net\dns_slave.h"
 				>
 			</File>
+			<File
+				RelativePath=".\net\sdch_dictionary_fetcher.cc"
+				>
+			</File>
+			<File
+				RelativePath=".\net\sdch_dictionary_fetcher.h"
+				>
+			</File>
 		</Filter>
 		<Filter
 			Name="RLZ"
diff --git a/chrome/browser/browser_main.cc b/chrome/browser/browser_main.cc
index 15703a2..fbb216a9 100644
--- a/chrome/browser/browser_main.cc
+++ b/chrome/browser/browser_main.cc
@@ -30,6 +30,7 @@
 #include "chrome/browser/jankometer.h"
 #include "chrome/browser/metrics_service.h"
 #include "chrome/browser/net/dns_global.h"
+#include "chrome/browser/net/sdch_dictionary_fetcher.h"
 #include "chrome/browser/plugin_service.h"
 #include "chrome/browser/printing/print_job_manager.h"
 #include "chrome/browser/rlz/rlz.h"
@@ -53,6 +54,7 @@
 #include "net/base/net_module.h"
 #include "net/base/net_resources.h"
 #include "net/base/net_util.h"
+#include "net/base/sdch_manager.h"
 #include "net/base/winsock_init.h"
 #include "net/http/http_network_layer.h"
 
@@ -475,6 +477,15 @@ int BrowserMain(CommandLine &parsed_command_line, int show_command,
   // Initialize the CertStore.
   CertStore::Initialize();
 
+  // Prepare for memory caching of SDCH dictionaries.
+  SdchManager sdch_manager;  // Construct singleton database.
+  if (parsed_command_line.HasSwitch(switches::kSdchFilter)) {
+    sdch_manager.set_sdch_fetcher(new SdchDictionaryFetcher);
+    std::wstring switch_domain =
+        parsed_command_line.GetSwitchValue(switches::kSdchFilter);
+    sdch_manager.enable_sdch_support(WideToASCII(switch_domain));
+  }
+
   MetricsService* metrics = NULL;
   if (!parsed_command_line.HasSwitch(switches::kDisableMetrics)) {
     if (parsed_command_line.HasSwitch(switches::kDisableMetricsReporting)) {
diff --git a/chrome/browser/net/sdch_dictionary_fetcher.cc b/chrome/browser/net/sdch_dictionary_fetcher.cc
new file mode 100644
index 0000000..1b5c21c
--- /dev/null
+++ b/chrome/browser/net/sdch_dictionary_fetcher.cc
@@ -0,0 +1,45 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/net/sdch_dictionary_fetcher.h"
+#include "chrome/browser/profile.h"
+
+void SdchDictionaryFetcher::Schedule(const GURL& dictionary_url) {
+  fetch_queue_.push(dictionary_url);
+  ScheduleDelayedRun();
+}
+
+// TODO(jar): If QOS low priority is supported, switch to using that instead of
+// just waiting to do the fetch.
+void SdchDictionaryFetcher::ScheduleDelayedRun() {
+  if (fetch_queue_.empty() || current_fetch_.get() || task_is_pending_)
+    return;
+  MessageLoop::current()->PostDelayedTask(FROM_HERE,
+      method_factory_.NewRunnableMethod(&SdchDictionaryFetcher::StartFetching),
+      kMsDelayFromRequestTillDownload);
+  task_is_pending_ = true;
+}
+
+void SdchDictionaryFetcher::StartFetching() {
+  DCHECK(task_is_pending_);
+  task_is_pending_ = false;
+
+  current_fetch_.reset(new URLFetcher(fetch_queue_.front(), URLFetcher::GET,
+                                      this));
+  fetch_queue_.pop();
+  current_fetch_->set_request_context(Profile::GetDefaultRequestContext());
+  current_fetch_->Start();
+}
+
+void SdchDictionaryFetcher::OnURLFetchComplete(const URLFetcher* source,
+                                               const GURL& url,
+                                               const URLRequestStatus& status,
+                                               int response_code,
+                                               const ResponseCookies& cookies,
+                                               const std::string& data) {
+  if (200 == response_code)
+    SdchManager::Global()->AddSdchDictionary(data, url);
+  current_fetch_.reset(NULL);
+  ScheduleDelayedRun();
+}
diff --git a/chrome/browser/net/sdch_dictionary_fetcher.h b/chrome/browser/net/sdch_dictionary_fetcher.h
new file mode 100644
index 0000000..34d3f28
--- /dev/null
+++ b/chrome/browser/net/sdch_dictionary_fetcher.h
@@ -0,0 +1,66 @@
+// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Support modularity by calling to load a new SDCH filter dictionary.
+// Note that this sort of calling can't be done in the /net directory, as it has
+// no concept of the HTTP cache (which is only visible at the browser level).
+
+#ifndef CHROME_BROWSER_NET_SDCH_DICTIONARY_FETCHER_H_
+#define CHROME_BROWSER_NET_SDCH_DICTIONARY_FETCHER_H_
+
+#include <queue>
+#include <string>
+
+#include "base/task.h"
+#include "chrome/browser/url_fetcher.h"
+#include "net/base/sdch_manager.h"
+
+class SdchDictionaryFetcher : public URLFetcher::Delegate,
+                              public SdchFetcher {
+ public:
+  #pragma warning(suppress: 4355)  // OK to pass "this" here.
+  SdchDictionaryFetcher() : method_factory_(this), task_is_pending_(false) {}
+  virtual ~SdchDictionaryFetcher() {}
+
+  // Implementation of SdchFetcher class.
+  // This method gets the requested dictionary, and then calls back into the
+  // SdchManager class with the dictionary's text.
+  virtual void Schedule(const GURL& dictionary_url);
+
+ private:
+  // Delay between Schedule and actual download.
+  static const int kMsDelayFromRequestTillDownload = 15000;
+
+  // Ensure the download after the above delay.
+  void ScheduleDelayedRun();
+
+  // Make sure we're processing (or waiting for) the the arrival of the next URL
+  // in the  |fetch_queue_|.
+  void StartFetching();
+
+  // Implementation of URLFetcher::Delegate. Called after transmission
+  // completes (either successfully or with failure).
+  virtual void OnURLFetchComplete(const URLFetcher* source,
+                                  const GURL& url,
+                                  const URLRequestStatus& status,
+                                  int response_code,
+                                  const ResponseCookies& cookies,
+                                  const std::string& data);
+
+  // A queue of URLs that are being used to download dictionaries.
+  std::queue<GURL> fetch_queue_;
+  // The currently outstanding URL fetch of a dicitonary.
+  // If this is null, then there is no outstanding request.
+  scoped_ptr<URLFetcher> current_fetch_;
+
+  // Always spread out the dictionary fetches, so that they don't steal
+  // bandwidth from the actual page load.  Create delayed tasks to spread out
+  // the download.
+  ScopedRunnableMethodFactory<SdchDictionaryFetcher> method_factory_;
+  bool task_is_pending_;
+
+  DISALLOW_COPY_AND_ASSIGN(SdchDictionaryFetcher);
+};
+
+#endif  // CHROME_BROWSER_NET_SDCH_DICTIONARY_FETCHER_H_
diff --git a/chrome/chrome.sln b/chrome/chrome.sln
index 246e950..3423cf4 100644
--- a/chrome/chrome.sln
+++ b/chrome/chrome.sln
@@ -162,6 +162,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "chrome_dll", "app\chrome_dl
 		{EF5E94AB-B646-4E5B-A058-52EF07B8351C} = {EF5E94AB-B646-4E5B-A058-52EF07B8351C}
 		{EFBB1436-A63F-4CD8-9E99-B89226E782EC} = {EFBB1436-A63F-4CD8-9E99-B89226E782EC}
 		{F4F4BCAA-EA59-445C-A119-3E6C29647A51} = {F4F4BCAA-EA59-445C-A119-3E6C29647A51}
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0} = {F54ABC59-5C00-414A-A9BA-BAF26D1699F0}
 		{FA537565-7B03-4FFC-AF15-F7A979B72E22} = {FA537565-7B03-4FFC-AF15-F7A979B72E22}
 		{FC0E1FD0-5DD7-4041-A1C9-CD3C376E4EED} = {FC0E1FD0-5DD7-4041-A1C9-CD3C376E4EED}
 	EndProjectSection
@@ -508,6 +509,7 @@ EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "net", "..\net\build\net.vcproj", "{326E9795-E760-410A-B69A-3F79DB3F5243}"
 	ProjectSection(ProjectDependencies) = postProject
 		{E13045CD-7E1F-4A41-9B18-8D288B2E7B41} = {E13045CD-7E1F-4A41-9B18-8D288B2E7B41}
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0} = {F54ABC59-5C00-414A-A9BA-BAF26D1699F0}
 	EndProjectSection
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tld_cleanup", "..\net\build\tld_cleanup.vcproj", "{E13045CD-7E1F-4A41-9B18-8D288B2E7B41}"
@@ -527,6 +529,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "net_unittests", "..\net\bui
 		{8C27D792-2648-4F5E-9ED0-374276327308} = {8C27D792-2648-4F5E-9ED0-374276327308}
 		{BFE8E2A7-3B3B-43B0-A994-3058B852DB8B} = {BFE8E2A7-3B3B-43B0-A994-3058B852DB8B}
 		{EF5E94AB-B646-4E5B-A058-52EF07B8351C} = {EF5E94AB-B646-4E5B-A058-52EF07B8351C}
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0} = {F54ABC59-5C00-414A-A9BA-BAF26D1699F0}
 	EndProjectSection
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "net_perftests", "..\net\build\net_perftests.vcproj", "{AAC78796-B9A2-4CD9-BF89-09B03E92BF73}"
@@ -1122,6 +1125,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "v8_shell_sample", "..\v8\to
 		{C0334F9A-1168-4101-9DD8-C30FB252D435} = {C0334F9A-1168-4101-9DD8-C30FB252D435}
 	EndProjectSection
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sdch", "..\sdch\sdch.vcproj", "{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -2431,6 +2436,16 @@ Global
 		{F4F4BCAA-EA59-445C-A119-3E6C29647A51}.Release|Mixed Platforms.Build.0 = Release|Win32
 		{F4F4BCAA-EA59-445C-A119-3E6C29647A51}.Release|Win32.ActiveCfg = Release|Win32
 		{F4F4BCAA-EA59-445C-A119-3E6C29647A51}.Release|Win32.Build.0 = Release|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Debug|Mixed Platforms.Build.0 = Debug|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Debug|Win32.ActiveCfg = Debug|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Debug|Win32.Build.0 = Debug|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Release|Any CPU.ActiveCfg = Release|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Release|Win32.ActiveCfg = Release|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Release|Win32.Build.0 = Release|Win32
 		{F7790A54-4078-4E4A-8231-818BE9FB1F94}.Debug|Any CPU.ActiveCfg = Debug|Win32
 		{F7790A54-4078-4E4A-8231-818BE9FB1F94}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
 		{F7790A54-4078-4E4A-8231-818BE9FB1F94}.Debug|Mixed Platforms.Build.0 = Debug|Win32
@@ -2641,6 +2656,7 @@ Global
 		{EF5E94AB-B646-4E5B-A058-52EF07B8351C} = {EF78C1F9-AA17-4CA5-B6CB-39B37A73A3DA}
 		{EFBB1436-A63F-4CD8-9E99-B89226E782EC} = {EB684A4B-98F7-4E68-8EA7-EA74ACF7060B}
 		{F4F4BCAA-EA59-445C-A119-3E6C29647A51} = {CB43561E-A6F8-49E2-96A2-3F2BA1FFF21E}
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0} = {EF78C1F9-AA17-4CA5-B6CB-39B37A73A3DA}
 		{F7790A54-4078-4E4A-8231-818BE9FB1F94} = {2325D8C4-8EF5-42AC-8900-492225750DE4}
 		{F9810DE8-CBC3-4605-A7B1-ECA2D5292FD7} = {032541FB-1E7C-4423-B657-4A71FE180C8A}
 		{FA39524D-3067-4141-888D-28A86C66F2B9} = {1174D37F-6ABB-45DA-81B3-C631281273B7}
diff --git a/chrome/chrome_kjs.sln b/chrome/chrome_kjs.sln
index ab358f4..b9f37dd 100644
--- a/chrome/chrome_kjs.sln
+++ b/chrome/chrome_kjs.sln
@@ -164,6 +164,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "chrome_dll", "app\chrome_dl
 		{EF5E94AB-B646-4E5B-A058-52EF07B8351C} = {EF5E94AB-B646-4E5B-A058-52EF07B8351C}
 		{EFBB1436-A63F-4CD8-9E99-B89226E782EC} = {EFBB1436-A63F-4CD8-9E99-B89226E782EC}
 		{F4F4BCAA-EA59-445C-A119-3E6C29647A51} = {F4F4BCAA-EA59-445C-A119-3E6C29647A51}
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0} = {F54ABC59-5C00-414A-A9BA-BAF26D1699F0}
 		{FA537565-7B03-4FFC-AF15-F7A979B72E22} = {FA537565-7B03-4FFC-AF15-F7A979B72E22}
 		{FC0E1FD0-5DD7-4041-A1C9-CD3C376E4EED} = {FC0E1FD0-5DD7-4041-A1C9-CD3C376E4EED}
 	EndProjectSection
@@ -512,6 +513,7 @@ EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "net", "..\net\build\net.vcproj", "{326E9795-E760-410A-B69A-3F79DB3F5243}"
 	ProjectSection(ProjectDependencies) = postProject
 		{E13045CD-7E1F-4A41-9B18-8D288B2E7B41} = {E13045CD-7E1F-4A41-9B18-8D288B2E7B41}
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0} = {F54ABC59-5C00-414A-A9BA-BAF26D1699F0}
 	EndProjectSection
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tld_cleanup", "..\net\build\tld_cleanup.vcproj", "{E13045CD-7E1F-4A41-9B18-8D288B2E7B41}"
@@ -1071,6 +1073,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "browser_views", "browser\vi
 		{D9DDAF60-663F-49CC-90DC-3D08CC3D1B28} = {D9DDAF60-663F-49CC-90DC-3D08CC3D1B28}
 	EndProjectSection
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sdch", "..\sdch\sdch.vcproj", "{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -2320,6 +2324,16 @@ Global
 		{F4F4BCAA-EA59-445C-A119-3E6C29647A51}.Release|Mixed Platforms.Build.0 = Release|Win32
 		{F4F4BCAA-EA59-445C-A119-3E6C29647A51}.Release|Win32.ActiveCfg = Release|Win32
 		{F4F4BCAA-EA59-445C-A119-3E6C29647A51}.Release|Win32.Build.0 = Release|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Debug|Mixed Platforms.Build.0 = Debug|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Debug|Win32.ActiveCfg = Debug|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Debug|Win32.Build.0 = Debug|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Release|Any CPU.ActiveCfg = Release|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Release|Win32.ActiveCfg = Release|Win32
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0}.Release|Win32.Build.0 = Release|Win32
 		{F7790A54-4078-4E4A-8231-818BE9FB1F94}.Debug|Any CPU.ActiveCfg = Debug|Win32
 		{F7790A54-4078-4E4A-8231-818BE9FB1F94}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
 		{F7790A54-4078-4E4A-8231-818BE9FB1F94}.Debug|Mixed Platforms.Build.0 = Debug|Win32
@@ -2524,6 +2538,7 @@ Global
 		{EF5E94AB-B646-4E5B-A058-52EF07B8351C} = {EF78C1F9-AA17-4CA5-B6CB-39B37A73A3DA}
 		{EFBB1436-A63F-4CD8-9E99-B89226E782EC} = {EB684A4B-98F7-4E68-8EA7-EA74ACF7060B}
 		{F4F4BCAA-EA59-445C-A119-3E6C29647A51} = {CB43561E-A6F8-49E2-96A2-3F2BA1FFF21E}
+		{F54ABC59-5C00-414A-A9BA-BAF26D1699F0} = {EF78C1F9-AA17-4CA5-B6CB-39B37A73A3DA}
 		{F7790A54-4078-4E4A-8231-818BE9FB1F94} = {2325D8C4-8EF5-42AC-8900-492225750DE4}
 		{F9810DE8-CBC3-4605-A7B1-ECA2D5292FD7} = {BBD3C9B1-AC02-41F4-AB89-D01B70009795}
 		{FA39524D-3067-4141-888D-28A86C66F2B9} = {1174D37F-6ABB-45DA-81B3-C631281273B7}
diff --git a/chrome/common/chrome_switches.cc b/chrome/common/chrome_switches.cc
index 56a3936..7dec663 100644
--- a/chrome/common/chrome_switches.cc
+++ b/chrome/common/chrome_switches.cc
@@ -317,5 +317,12 @@ const wchar_t kJavaScriptDebuggerPath[] = L"javascript-debugger-path";
 
 const wchar_t kEnableP13n[] = L"enable-p13n";
 
+// Enable support for SDCH filtering (dictionary based expansion of content).
+// Optional argument is *the* only domain name that will have SDCH suppport.
+// Default is  "-enable-sdch" to advertise SDCH on all domains.
+// Sample usage with argument: "-enable-sdch=.google.com"
+// SDCH is currently only supported server-side for searches on google.com.
+const wchar_t kSdchFilter[] = L"enable-sdch";
+
 }  // namespace switches
 
diff --git a/chrome/common/chrome_switches.h b/chrome/common/chrome_switches.h
index 4ec96f4..241170e8 100644
--- a/chrome/common/chrome_switches.h
+++ b/chrome/common/chrome_switches.h
@@ -123,6 +123,8 @@ extern const wchar_t kJavaScriptDebuggerPath[];
 
 extern const wchar_t kEnableP13n[];
 
+extern const wchar_t kSdchFilter[];
+
 }  // namespace switches
 
 #endif  // CHROME_COMMON_CHROME_SWITCHES_H__