See if 'charset' is available in HTTP response header and use that to convert the response to UTF-8 before

sending over to JSON deserializer. (previously this issue was tracked internally as issue 1293145)


BUG=2806
TEST=In French Chrome, set your default search engine to Voila and begin to type a word with an accented letters and see
if 'Search Voila for ' contains the word (it's not a bullet-proof test).
In Korean chrome, set the default search engine to Daum and begin to type a Korean word (with Korean IME on, type 'qk').


Review URL: http://codereview.chromium.org/4287

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@2623 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 24 insertions, 19 deletions

diff --git a/chrome/browser/autocomplete/search_provider.cc b/chrome/browser/autocomplete/search_provider.cc
index 5ec75d2..71b1ef5 100644
--- a/chrome/browser/autocomplete/search_provider.cc
+++ b/chrome/browser/autocomplete/search_provider.cc
@@ -110,7 +110,24 @@ void SearchProvider::OnURLFetchComplete(const URLFetcher* source,
   suggest_results_pending_ = false;
   suggest_results_.clear();
   navigation_results_.clear();
-  JSONStringValueSerializer deserializer(data);
+  const net::HttpResponseHeaders* const response_headers =
+      source->response_headers();
+  std::string json_data(data);
+  // JSON is supposed to be in UTF-8, but some suggest service
+  // providers send JSON files in non-UTF-8 encodings, but they're 
+  // usually correctly specified in Content-Type header field.
+  if (response_headers) {
+    std::string charset;
+    if (response_headers->GetCharset(&charset)) {
+      std::wstring wide_data;
+      // TODO(jungshik): Switch to CodePageToUTF8 after it's added.
+      if (CodepageToWide(data, charset.c_str(),
+                         OnStringUtilConversionError::FAIL, &wide_data))
+        json_data = WideToUTF8(wide_data); 
+    }
+  }
+
+  JSONStringValueSerializer deserializer(json_data);
   deserializer.set_allow_trailing_comma(true);
   Value* root_val = NULL;
   have_suggest_results_ = status.is_success() && (response_code == 200) &&
diff --git a/chrome/browser/template_url_prepopulate_data.cc b/chrome/browser/template_url_prepopulate_data.cc
index ca92cd4..0f8e299 100644
--- a/chrome/browser/template_url_prepopulate_data.cc
+++ b/chrome/browser/template_url_prepopulate_data.cc
@@ -331,11 +331,7 @@ const PrepopulatedEngine daum = {
   L"http://search.daum.net/favicon.ico",
   L"http://search.daum.net/search?q={searchTerms}",
   "EUC-KR",
-  // Response is in EUC-KR and is labelled as such in HTTP C-T header.
-  // L"http://sug.search.daum.net/search_nsuggest?mod=fxjson&q={searchTerms}",
-  // Until http://b/1293145 is fixed or we figure out how to get responses
-  // in UTF-8, disable it. 
-  NULL,
+   L"http://sug.search.daum.net/search_nsuggest?mod=fxjson&q={searchTerms}",
   68,
 };
 
@@ -1417,12 +1413,8 @@ const PrepopulatedEngine orange = {
   L"http://www.orange.fr/favicon.ico",
   L"http://rws.search.ke.voila.fr/RW/S/opensearch_orange?rdata={searchTerms}",
   "ISO-8859-1",
-  // Response is in ISO-8859-1 and is labelled as such in HTTP C-T header.
-  // L"http://search.ke.voila.fr/fr/cmplopensearch/xml/fullxml?"
-  // L"rdata={searchTerms}",
-  // Until http://b/1293145 is fixed or we figure out how to get responses
-  // in UTF-8, disable it. 
-  NULL,
+  L"http://search.ke.voila.fr/fr/cmplopensearch/xml/fullxml?"
+  L"rdata={searchTerms}",
   48,
 };
 
@@ -1746,12 +1738,8 @@ const PrepopulatedEngine voila = {
   L"http://search.ke.voila.fr/favicon.ico",
   L"http://rws.search.ke.voila.fr/RW/S/opensearch_voila?rdata={searchTerms}",
   "ISO-8859-1",
-  // Response is in ISO-8859-1 and is labelled as such in HTTP C-T header.
-  // L"http://search.ke.voila.fr/fr/cmplopensearch/xml/fullxml?"
-  // L"rdata={searchTerms}",
-  // Until http://b/1293145 is fixed or we figure out how to get responses
-  // in UTF-8, disable it. 
-  NULL,
+  L"http://search.ke.voila.fr/fr/cmplopensearch/xml/fullxml?"
+  L"rdata={searchTerms}",
   47,
 };
 
@@ -3016,7 +3004,7 @@ void RegisterUserPrefs(PrefService* prefs) {
 }
 
 int GetDataVersion() {
-  return 14;  // Increment this if you change the above data in ways that mean
+  return 15;  // Increment this if you change the above data in ways that mean
              // users with existing data should get a new version.
 }