Convert Chinese lang codes with a region code to a compatible lang code

Convert zh-HK, zh-MO, zh-SG to their compatible language codes such as
zh-CN and zh-TW. This will help to show Translate UI on websites in
the Chinese langs with the region codes.

BUG=409606
TEST=components_unittests --gtest_filter=Language*:Translate*, unit_tests --gtest_filter=Translate*, browser_tests --gtest_filter=ChromeRenderViewTest.LanguageCommonMistakesAreCorrected

Review URL: https://codereview.chromium.org/715593002

Cr-Commit-Position: refs/heads/master@{#303981}

7 files changed, 43 insertions, 65 deletions

diff --git a/components/translate/core/browser/translate_manager.cc b/components/translate/core/browser/translate_manager.cc
index 6fa1aa4..065e102 100644
--- a/components/translate/core/browser/translate_manager.cc
+++ b/components/translate/core/browser/translate_manager.cc
@@ -28,6 +28,7 @@
 #include "components/translate/core/common/translate_constants.h"
 #include "components/translate/core/common/translate_pref_names.h"
 #include "components/translate/core/common/translate_switches.h"
+#include "components/translate/core/common/translate_util.h"
 #include "net/base/url_util.h"
 #include "net/http/http_status_code.h"
 
@@ -350,9 +351,9 @@ void TranslateManager::OnTranslateScriptFetchComplete(
 // static
 std::string TranslateManager::GetTargetLanguage(
     const std::vector<std::string>& accept_languages_list) {
-  std::string ui_lang = TranslatePrefs::ConvertLangCodeForTranslation(
-      TranslateDownloadManager::GetLanguageCode(
-          TranslateDownloadManager::GetInstance()->application_locale()));
+  std::string ui_lang = TranslateDownloadManager::GetLanguageCode(
+      TranslateDownloadManager::GetInstance()->application_locale());
+  translate::ToTranslateLanguageSynonym(&ui_lang);
 
   if (TranslateDownloadManager::IsSupportedLanguage(ui_lang))
     return ui_lang;
diff --git a/components/translate/core/browser/translate_prefs.cc b/components/translate/core/browser/translate_prefs.cc
index 4f9c2a7..a7cd5c1 100644
--- a/components/translate/core/browser/translate_prefs.cc
+++ b/components/translate/core/browser/translate_prefs.cc
@@ -492,38 +492,21 @@ void TranslatePrefs::CreateBlockedLanguages(
 
   for (std::vector<std::string>::const_iterator it = accept_languages.begin();
        it != accept_languages.end(); ++it) {
-    std::string converted_lang = ConvertLangCodeForTranslation(*it);
+    std::string lang = *it;
+    translate::ToTranslateLanguageSynonym(&lang);
 
     // Regarding http://crbug.com/36182, even though English exists in Accept
     // language list, English could be translated on non-English locale.
-    if (converted_lang == "en" && !is_ui_english)
+    if (lang == "en" && !is_ui_english)
       continue;
 
-    result.insert(converted_lang);
+    result.insert(lang);
   }
 
   blocked_languages->insert(
       blocked_languages->begin(), result.begin(), result.end());
 }
 
-// static
-std::string TranslatePrefs::ConvertLangCodeForTranslation(
-    const std::string& lang) {
-  std::vector<std::string> tokens;
-  base::SplitString(lang, '-', &tokens);
-  if (tokens.size() < 1)
-    return lang;
-
-  std::string main_part = tokens[0];
-
-  // Translate doesn't support General Chinese and the sub code is necessary.
-  if (main_part == "zh")
-    return lang;
-
-  translate::ToTranslateLanguageSynonym(&main_part);
-  return main_part;
-}
-
 bool TranslatePrefs::IsValueInList(const base::ListValue* list,
                                    const std::string& in_value) const {
   for (size_t i = 0; i < list->GetSize(); ++i) {
diff --git a/components/translate/core/browser/translate_prefs.h b/components/translate/core/browser/translate_prefs.h
index c1b89c5..8625ee3 100644
--- a/components/translate/core/browser/translate_prefs.h
+++ b/components/translate/core/browser/translate_prefs.h
@@ -115,12 +115,6 @@ class TranslatePrefs {
   static void MigrateUserPrefs(PrefService* user_prefs,
                                const char* accept_languages_pref);
 
-  // Converts the language code for Translate. This removes the sub code (like
-  // -US) except for Chinese, and converts the synonyms.
-  // The same logic exists at language_options.js, and please keep consistency
-  // with the JavaScript file.
-  static std::string ConvertLangCodeForTranslation(const std::string& lang);
-
  private:
   friend class TranslatePrefsTest;
   FRIEND_TEST_ALL_PREFIXES(TranslatePrefsTest, CreateBlockedLanguages);
diff --git a/components/translate/core/common/translate_util.cc b/components/translate/core/common/translate_util.cc
index 6a73b11..2c30489 100644
--- a/components/translate/core/common/translate_util.cc
+++ b/components/translate/core/common/translate_util.cc
@@ -62,6 +62,17 @@ const LanguageCodePair kLanguageCodeSynonyms[] = {
   {"jw", "jv"},
 };
 
+// Some Chinese language codes are compatible with zh-TW or zh-CN in terms of
+// Translate.
+//
+// If this table is updated, please sync this with the synonym table in
+// chrome/browser/resources/options/language_options.js
+const LanguageCodePair kLanguageCodeChineseCompatiblePairs[] = {
+  {"zh-TW", "zh-HK"},
+  {"zh-TW", "zh-MO"},
+  {"zh-CN", "zh-SG"},
+};
+
 const char kSecurityOrigin[] = "https://translate.googleapis.com/";
 
 void ToTranslateLanguageSynonym(std::string* language) {
@@ -72,6 +83,13 @@ void ToTranslateLanguageSynonym(std::string* language) {
     }
   }
 
+  for (size_t i = 0; i < arraysize(kLanguageCodeChineseCompatiblePairs); ++i) {
+    if (*language == kLanguageCodeChineseCompatiblePairs[i].chrome_language) {
+      *language = kLanguageCodeChineseCompatiblePairs[i].translate_language;
+      return;
+    }
+  }
+
   std::string main_part, tail_part;
   SplitIntoMainAndTail(*language, &main_part, &tail_part);
   if (main_part.empty())
@@ -85,7 +103,12 @@ void ToTranslateLanguageSynonym(std::string* language) {
     }
   }
 
-  *language = main_part + tail_part;
+  if (main_part == "zh") {
+    *language = main_part + tail_part;
+    return;
+  }
+
+  *language = main_part;
 }
 
 void ToChromeLanguageSynonym(std::string* language) {
diff --git a/components/translate/core/common/translate_util.h b/components/translate/core/common/translate_util.h
index dcabd2a..d7548dd 100644
--- a/components/translate/core/common/translate_util.h
+++ b/components/translate/core/common/translate_util.h
@@ -15,6 +15,9 @@ namespace translate {
 extern const char kSecurityOrigin[];
 
 // Converts language code synonym to use at Translate server.
+//
+// The same logic exists at language_options.js, and please keep consistency
+// with the JavaScript file.
 void ToTranslateLanguageSynonym(std::string* language);
 
 // Converts language code synonym to use at Chrome internal.
diff --git a/components/translate/core/common/translate_util_unittest.cc b/components/translate/core/common/translate_util_unittest.cc
index d42df2c..e3754ff 100644
--- a/components/translate/core/common/translate_util_unittest.cc
+++ b/components/translate/core/common/translate_util_unittest.cc
@@ -19,27 +19,18 @@ TEST_F(TranslateUtilTest, ToTranslateLanguageSynonym) {
   translate::ToTranslateLanguageSynonym(&language);
   EXPECT_EQ("no", language);
 
-  language = std::string("he");
+  language = std::string("zh-HK");
   translate::ToTranslateLanguageSynonym(&language);
-  EXPECT_EQ("iw", language);
-
-  language = std::string("jv");
-  translate::ToTranslateLanguageSynonym(&language);
-  EXPECT_EQ("jw", language);
+  EXPECT_EQ("zh-TW", language);
 
-  language = std::string("fil");
-  translate::ToTranslateLanguageSynonym(&language);
-  EXPECT_EQ("tl", language);
-
-  // Preserve a sub code if the language has a synonym.
+  // A sub code is not preserved (except for Chinese).
   language = std::string("he-IL");
   translate::ToTranslateLanguageSynonym(&language);
-  EXPECT_EQ("iw-IL", language);
+  EXPECT_EQ("iw", language);
 
-  // Don't preserve a sub code if the language has just a similitude.
-  language = std::string("nb-NO");
+  language = std::string("zh-JP");
   translate::ToTranslateLanguageSynonym(&language);
-  EXPECT_EQ("nb-NO", language);
+  EXPECT_EQ("zh-JP", language);
 
   // Preserve the argument if it doesn't have its synonym.
   language = std::string("en");
@@ -55,28 +46,11 @@ TEST_F(TranslateUtilTest, ToChromeLanguageSynonym) {
   translate::ToChromeLanguageSynonym(&language);
   EXPECT_EQ("nb", language);
 
-  language = std::string("iw");
-  translate::ToChromeLanguageSynonym(&language);
-  EXPECT_EQ("he", language);
-
-  language = std::string("jw");
-  translate::ToChromeLanguageSynonym(&language);
-  EXPECT_EQ("jv", language);
-
-  language = std::string("tl");
-  translate::ToChromeLanguageSynonym(&language);
-  EXPECT_EQ("fil", language);
-
-  // Preserve a sub code if the language has a synonym.
+  // Preserve a sub code
   language = std::string("iw-IL");
   translate::ToChromeLanguageSynonym(&language);
   EXPECT_EQ("he-IL", language);
 
-  // Don't preserve a sub code if the language has just a similitude.
-  language = std::string("no-NO");
-  translate::ToChromeLanguageSynonym(&language);
-  EXPECT_EQ("no-NO", language);
-
   // Preserve the argument if it doesn't have its synonym.
   language = std::string("en");
   translate::ToChromeLanguageSynonym(&language);
diff --git a/components/translate/core/language_detection/language_detection_util_unittest.cc b/components/translate/core/language_detection/language_detection_util_unittest.cc
index 135a759..3103458 100644
--- a/components/translate/core/language_detection/language_detection_util_unittest.cc
+++ b/components/translate/core/language_detection/language_detection_util_unittest.cc
@@ -126,7 +126,7 @@ TEST_F(LanguageDetectionUtilTest, CLDAgreeWithLanguageCodeHavingCountryCode) {
                                                           contents,
                                                           &cld_language,
                                                           &is_cld_reliable);
-  EXPECT_EQ("en-US", language);
+  EXPECT_EQ("en", language);
   EXPECT_EQ("en", cld_language);
   EXPECT_TRUE(is_cld_reliable);
 }