Implementing better fallback algorithm.

Before:
 current_locale->default_locale

Now:
 current_locale->chain_of_parent_locales->default_locale

If default_locale is de, and current locale en_US, we follow:
en_US -> en -> de

en is not a Chrome locale (only en_US, en_GB are), but we fake it to allow this kind of fallback.

Developers can implement common locale root with most of the messages (like en) and put locale specifics in en_GB (color->colour) or en_US. You can even symlink en and en_US and save on work.

I am planning on fixing loading local resources too, to use this child->parent fallback.

BUG=12131

Review URL: http://codereview.chromium.org/293037

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@29684 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 132 insertions, 48 deletions

diff --git a/chrome/browser/extensions/extension_file_util.cc b/chrome/browser/extensions/extension_file_util.cc
index 0bd9507..23bd085 100644
--- a/chrome/browser/extensions/extension_file_util.cc
+++ b/chrome/browser/extensions/extension_file_util.cc
@@ -362,6 +362,7 @@ ExtensionMessageBundle* LoadLocaleInfo(const FilePath& extension_path,
     extension_l10n_util::LoadMessageCatalogs(locale_path,
                                              default_locale,
                                              app_locale,
+                                             locales,
                                              error);
   return message_bundle;
 }
diff --git a/chrome/browser/extensions/extension_l10n_util.cc b/chrome/browser/extensions/extension_l10n_util.cc
index 411de59..4c8b116 100644
--- a/chrome/browser/extensions/extension_l10n_util.cc
+++ b/chrome/browser/extensions/extension_l10n_util.cc
@@ -10,6 +10,7 @@
 
 #include "app/l10n_util.h"
 #include "base/file_util.h"
+#include "base/linked_ptr.h"
 #include "base/string_util.h"
 #include "base/values.h"
 #include "chrome/common/extensions/extension.h"
@@ -29,49 +30,85 @@ std::string GetDefaultLocaleFromManifest(const DictionaryValue& manifest,
     *error = errors::kInvalidDefaultLocale;
     return "";
   }
-  // Normalize underscores to hyphens.
-  std::replace(default_locale.begin(), default_locale.end(), '_', '-');
+
   return default_locale;
 }
 
 bool AddLocale(const std::set<std::string>& chrome_locales,
                const FilePath& locale_folder,
+               const std::string& locale_name,
                std::set<std::string>* valid_locales,
-               std::string* locale_name,
                std::string* error) {
-  // Normalize underscores to hyphens because that's what our locale files use.
-  std::replace(locale_name->begin(), locale_name->end(), '_', '-');
   // Accept name that starts with a . but don't add it to the list of supported
   // locales.
-  if (locale_name->find(".") == 0)
+  if (locale_name.find(".") == 0)
     return true;
-  if (chrome_locales.find(*locale_name) == chrome_locales.end()) {
+  if (chrome_locales.find(locale_name) == chrome_locales.end()) {
     // Fail if there is an extension locale that's not in the Chrome list.
     *error = StringPrintf("Supplied locale %s is not supported.",
-                          locale_name->c_str());
+                          locale_name.c_str());
     return false;
   }
   // Check if messages file is actually present (but don't check content).
   if (file_util::PathExists(
          locale_folder.AppendASCII(Extension::kMessagesFilename))) {
-    valid_locales->insert(*locale_name);
+    valid_locales->insert(locale_name);
   } else {
     *error = StringPrintf("Catalog file is missing for locale %s.",
-                          locale_name->c_str());
+                          locale_name.c_str());
     return false;
   }
 
   return true;
 }
 
+// Converts all - into _, to be consistent with ICU and file system names.
+static std::string NormalizeLocale(const std::string& locale) {
+  std::string normalized_locale(locale);
+  std::replace(normalized_locale.begin(), normalized_locale.end(), '-', '_');
+
+  return normalized_locale;
+}
+
+// Produce a vector of parent locales for given locale.
+// It includes the current locale in the result.
+// sr_Cyrl_RS generates sr_Cyrl_RS, sr_Cyrl and sr.
+static void GetParentLocales(const std::string& current_locale,
+                             std::vector<std::string>* parent_locales) {
+  std::string locale(NormalizeLocale(current_locale));
+
+  const int kNameCapacity = 256;
+  char parent[kNameCapacity];
+  strncpy(parent, locale.c_str(), kNameCapacity);
+  parent_locales->push_back(parent);
+  UErrorCode err = U_ZERO_ERROR;
+  while (uloc_getParent(parent, parent, kNameCapacity, &err) > 0) {
+    if (err != U_ZERO_ERROR)
+      break;
+    parent_locales->push_back(parent);
+  }
+}
+
+// Extends list of Chrome locales to them and their parents, so we can do
+// proper fallback.
+static void GetAllLocales(std::set<std::string>* all_locales) {
+  const std::vector<std::string>& available_locales =
+      l10n_util::GetAvailableLocales();
+  // Add all parents of the current locale to the available locales set.
+  // I.e. for sr_Cyrl_RS we add sr_Cyrl_RS, sr_Cyrl and sr.
+  for (size_t i = 0; i < available_locales.size(); ++i) {
+    std::vector<std::string> result;
+    GetParentLocales(available_locales[i], &result);
+    all_locales->insert(result.begin(), result.end());
+  }
+}
+
 bool GetValidLocales(const FilePath& locale_path,
                      std::set<std::string>* valid_locales,
                      std::string* error) {
-  // Get available chrome locales as a set.
-  const std::vector<std::string>& available_locales =
-      l10n_util::GetAvailableLocales();
-  static std::set<std::string> chrome_locales(available_locales.begin(),
-                                              available_locales.end());
+  static std::set<std::string> chrome_locales;
+  GetAllLocales(&chrome_locales);
+
   // Enumerate all supplied locales in the extension.
   file_util::FileEnumerator locales(locale_path,
                                     false,
@@ -82,8 +119,8 @@ bool GetValidLocales(const FilePath& locale_path,
         WideToASCII(locale_folder.BaseName().ToWStringHack());
     if (!AddLocale(chrome_locales,
                    locale_folder,
+                   locale_name,
                    valid_locales,
-                   &locale_name,
                    error)) {
       return false;
     }
@@ -103,9 +140,8 @@ bool GetValidLocales(const FilePath& locale_path,
 static DictionaryValue* LoadMessageFile(const FilePath& locale_path,
                                         const std::string& locale,
                                         std::string* error) {
+
   std::string extension_locale = locale;
-  // Normalize hyphens to underscores because that's what our locale files use.
-  std::replace(extension_locale.begin(), extension_locale.end(), '-', '_');
   FilePath file = locale_path.AppendASCII(extension_locale)
       .AppendASCII(Extension::kMessagesFilename);
   JSONFileValueSerializer messages_serializer(file);
@@ -124,24 +160,31 @@ ExtensionMessageBundle* LoadMessageCatalogs(
     const FilePath& locale_path,
     const std::string& default_locale,
     const std::string& application_locale,
+    const std::set<std::string>& valid_locales,
     std::string* error) {
-  scoped_ptr<DictionaryValue> default_catalog(
-      LoadMessageFile(locale_path, default_locale, error));
-  if (!default_catalog.get()) {
-    return false;
-  }
-
-  scoped_ptr<DictionaryValue> app_catalog(
-      LoadMessageFile(locale_path, application_locale, error));
-  if (!app_catalog.get()) {
-    // Only default catalog has to be present. This is not an error.
-    app_catalog.reset(new DictionaryValue);
-    error->clear();
+  // Order locales to load as current_locale, first_parent, ..., default_locale.
+  std::vector<std::string> all_fallback_locales;
+  if (!application_locale.empty() && application_locale != default_locale)
+    GetParentLocales(application_locale, &all_fallback_locales);
+  all_fallback_locales.push_back(default_locale);
+
+  std::vector<linked_ptr<DictionaryValue> > catalogs;
+  for (size_t i = 0; i < all_fallback_locales.size(); ++i) {
+    // Skip all parent locales that are not supplied.
+    if (valid_locales.find(all_fallback_locales[i]) == valid_locales.end())
+      continue;
+    linked_ptr<DictionaryValue> catalog(
+      LoadMessageFile(locale_path, all_fallback_locales[i], error));
+    if (!catalog.get()) {
+      // If locale is valid, but messages.json is corrupted or missing, return
+      // an error.
+      return false;
+    } else {
+      catalogs.push_back(catalog);
+    }
   }
 
-  return ExtensionMessageBundle::Create(*default_catalog,
-                                        *app_catalog,
-                                        error);
+  return ExtensionMessageBundle::Create(catalogs, error);
 }
 
 FilePath GetL10nRelativePath(const FilePath& relative_resource_path) {
diff --git a/chrome/browser/extensions/extension_l10n_util.h b/chrome/browser/extensions/extension_l10n_util.h
index b3be601..57c2033 100644
--- a/chrome/browser/extensions/extension_l10n_util.h
+++ b/chrome/browser/extensions/extension_l10n_util.h
@@ -29,8 +29,8 @@ std::string GetDefaultLocaleFromManifest(const DictionaryValue& manifest,
 // If file name starts with . return true (helps testing extensions under svn).
 bool AddLocale(const std::set<std::string>& chrome_locales,
                const FilePath& locale_folder,
+               const std::string& locale_name,
                std::set<std::string>* valid_locales,
-               std::string* locale_name,
                std::string* error);
 
 // Adds valid locales to the extension.
@@ -44,16 +44,17 @@ bool GetValidLocales(const FilePath& locale_path,
                      std::set<std::string>* locales,
                      std::string* error);
 
-// Loads messages file for default locale, and application locale (application
-// locale doesn't have to exist).
-// It creates simplified in-memory representation of name-value pairs, where
-// value part is actual message with placeholders resolved.
+// Loads messages file for default locale, and application locales (application
+// locales doesn't have to exist). Application locale is current locale and its
+// parents.
 // Returns message bundle if it can load default locale messages file, and all
 // messages are valid, else returns NULL and sets error.
-ExtensionMessageBundle* LoadMessageCatalogs(const FilePath& locale_path,
-                                            const std::string& default_locale,
-                                            const std::string& app_locale,
-                                            std::string* error);
+ExtensionMessageBundle* LoadMessageCatalogs(
+    const FilePath& locale_path,
+    const std::string& default_locale,
+    const std::string& app_locale,
+    const std::set<std::string>& valid_locales,
+    std::string* error);
 
 // Returns relative l10n path to the resource.
 FilePath GetL10nRelativePath(const FilePath& relative_resource_path);
diff --git a/chrome/browser/extensions/extension_l10n_util_unittest.cc b/chrome/browser/extensions/extension_l10n_util_unittest.cc
index 5c0df39..acf946c 100644
--- a/chrome/browser/extensions/extension_l10n_util_unittest.cc
+++ b/chrome/browser/extensions/extension_l10n_util_unittest.cc
@@ -66,8 +66,35 @@ TEST(ExtensionL10nUtil, GetValidLocalesWithValidLocalesAndMessagesFile) {
   EXPECT_TRUE(extension_l10n_util::GetValidLocales(install_dir,
                                                    &locales,
                                                    &error));
-  EXPECT_EQ(2U, locales.size());
+  EXPECT_EQ(3U, locales.size());
   EXPECT_TRUE(locales.find("sr") != locales.end());
+  EXPECT_TRUE(locales.find("en") != locales.end());
+  EXPECT_TRUE(locales.find("en_US") != locales.end());
+}
+
+TEST(ExtensionL10nUtil, LoadMessageCatalogsValidFallback) {
+  FilePath install_dir;
+  ASSERT_TRUE(PathService::Get(chrome::DIR_TEST_DATA, &install_dir));
+  install_dir = install_dir.AppendASCII("extensions")
+      .AppendASCII("good")
+      .AppendASCII("Extensions")
+      .AppendASCII("behllobkkfkfnphdnhnkndlbkcpglgmj")
+      .AppendASCII("1.0.0.0")
+      .AppendASCII(Extension::kLocaleFolder);
+
+  std::string error;
+  std::set<std::string> locales;
+  EXPECT_TRUE(extension_l10n_util::GetValidLocales(install_dir,
+                                                   &locales,
+                                                   &error));
+
+  scoped_ptr<ExtensionMessageBundle> bundle(
+      extension_l10n_util::LoadMessageCatalogs(
+          install_dir, "sr", "en_US", locales, &error));
+  ASSERT_FALSE(NULL == bundle.get());
+  EXPECT_TRUE(error.empty());
+  EXPECT_EQ("Color", bundle->GetL10nMessage("color"));
+  EXPECT_EQ("Not in the US or GB.", bundle->GetL10nMessage("not_in_US_or_GB"));
 }
 
 TEST(ExtensionL10nUtil, LoadMessageCatalogsMissingFiles) {
@@ -77,10 +104,14 @@ TEST(ExtensionL10nUtil, LoadMessageCatalogsMissingFiles) {
   FilePath src_path = temp.path().AppendASCII(Extension::kLocaleFolder);
   ASSERT_TRUE(file_util::CreateDirectory(src_path));
 
+  std::set<std::string> valid_locales;
+  valid_locales.insert("sr");
+  valid_locales.insert("en");
   std::string error;
   EXPECT_TRUE(NULL == extension_l10n_util::LoadMessageCatalogs(src_path,
-                                                               "en-US",
+                                                               "en",
                                                                "sr",
+                                                               valid_locales,
                                                                &error));
   EXPECT_FALSE(error.empty());
 }
@@ -92,7 +123,7 @@ TEST(ExtensionL10nUtil, LoadMessageCatalogsBadJSONFormat) {
   FilePath src_path = temp.path().AppendASCII(Extension::kLocaleFolder);
   ASSERT_TRUE(file_util::CreateDirectory(src_path));
 
-  FilePath locale = src_path.AppendASCII("en_US");
+  FilePath locale = src_path.AppendASCII("sr");
   ASSERT_TRUE(file_util::CreateDirectory(locale));
 
   std::string data = "{ \"name\":";
@@ -100,10 +131,14 @@ TEST(ExtensionL10nUtil, LoadMessageCatalogsBadJSONFormat) {
       file_util::WriteFile(locale.AppendASCII(Extension::kMessagesFilename),
                            data.c_str(), data.length()));
 
+  std::set<std::string> valid_locales;
+  valid_locales.insert("sr");
+  valid_locales.insert("en_US");
   std::string error;
   EXPECT_TRUE(NULL == extension_l10n_util::LoadMessageCatalogs(src_path,
-                                                              "en-US",
+                                                              "en_US",
                                                               "sr",
+                                                              valid_locales,
                                                               &error));
   EXPECT_EQ("Line: 1, column: 10, Syntax error.", error);
 }
@@ -115,7 +150,7 @@ TEST(ExtensionL10nUtil, LoadMessageCatalogsDuplicateKeys) {
   FilePath src_path = temp.path().AppendASCII(Extension::kLocaleFolder);
   ASSERT_TRUE(file_util::CreateDirectory(src_path));
 
-  FilePath locale_1 = src_path.AppendASCII("en_US");
+  FilePath locale_1 = src_path.AppendASCII("en");
   ASSERT_TRUE(file_util::CreateDirectory(locale_1));
 
   std::string data =
@@ -132,13 +167,17 @@ TEST(ExtensionL10nUtil, LoadMessageCatalogsDuplicateKeys) {
       file_util::WriteFile(locale_2.AppendASCII(Extension::kMessagesFilename),
                            data.c_str(), data.length()));
 
+  std::set<std::string> valid_locales;
+  valid_locales.insert("sr");
+  valid_locales.insert("en");
   std::string error;
   // JSON parser hides duplicates. We are going to get only one key/value
   // pair at the end.
   scoped_ptr<ExtensionMessageBundle> message_bundle(
       extension_l10n_util::LoadMessageCatalogs(src_path,
-                                               "en-US",
+                                               "en",
                                                "sr",
+                                               valid_locales,
                                                &error));
   EXPECT_TRUE(NULL != message_bundle.get());
   EXPECT_TRUE(error.empty());