Reland r36541 (which went in without any commit log by some magic). Was reverted in r36550. Now I'm relanding with the full description.

Port back CLD to Linux and Mac by replacing Windows API calls with ICU's equivalent APIs for normalization. 

I also fixed bug 23553 (Traditional Chinese is not detected) by calling LanguageCode instead of LanguageCode_ISO_639_1. The latter covers only ISO 639-1, but there are languages detected by CLD not covered by ISO 639-1. In that case, ISO 639-2 is used. In case even ISO 639-2 does not cover (e.g. Traditional Chinese), another fallback is taken by LanguageCode. 

The html file for CLD testing (french_sentence.html) is explicitly labelled with charset=ISO-8859-1.

Original Review: http://codereview.chromium.org/523108

BUG=25206,23553
TEST=1. CLD is built on Linux/Mac
     2. The following test pass:
       - unit_tests: Extension*.DetectTabLang* and CompactLangDet*.* 
       - browser_tests: ExtensionBrowserTest.Toolstrip 
     3. Install the 'cld extension' in chrome/common/extensions/docs/examples/api/i18n/cld and go to http://news.google.com.tw and 'zh-TW' shows up in the language badge at the upper right (upper-left in he/ar Chrome) corner. 
TBR=jcampan







Review URL: http://codereview.chromium.org/545123

TBR=jshin@chromium.org
Review URL: http://codereview.chromium.org/551070

TBR=jshin@chromium.org
Review URL: http://codereview.chromium.org/549091

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@36552 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 126 insertions, 112 deletions

diff --git a/third_party/cld/cld.gyp b/third_party/cld/cld.gyp
index 2f3f192..6913f2e 100644
--- a/third_party/cld/cld.gyp
+++ b/third_party/cld/cld.gyp
@@ -3,119 +3,133 @@
 # found in the LICENSE file.
 
 {
-  'conditions': [
-    ['OS=="win"', {
-      'targets': [
-        {
-          'target_name': 'cld',
-          'type': '<(library)',
-          'include_dirs': [
-            '.',
-          ],
-          'msvs_disabled_warnings': [4005, 4006, 4018, 4244, 4309, 4800],
-          'defines': [
-            'CLD_WINDOWS',
-          ],
-          'sources': [
-            'bar/common/scopedlibrary.h',
-            'bar/common/scopedptr.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/cldutil.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/cldutil.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/cldutil_dbg.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/cldutil_dbg_empty.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/compact_lang_det.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/compact_lang_det.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/compact_lang_det_impl.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/compact_lang_det_impl.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/ext_lang_enc.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/ext_lang_enc.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/getonescriptspan.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/getonescriptspan.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/letterscript_enum.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/letterscript_enum.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/subsetsequence.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/subsetsequence.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/tote.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/tote.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/utf8propjustletter.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/utf8propletterscriptnum.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/utf8scannotjustletterspecial.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_longwords8_0.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_meanscore.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_quads_128.cc',
-            # For now using the 128 bytes detection in order to save hundreds of KBs on the final package.
-            # 'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_quads_256.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_basictypes.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_commandlineflags.h',
-            # We use the static table at this point, so we don't need to compile the following files:
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_dynamicstate.h',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_dynamicstate.cc',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_loadpolicy.cc',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_loadpolicy.h',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_loadpolicyinterface.h',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_resourceids.h',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_service.h',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_service.cc',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_serviceinterface.h',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_tables.cc',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_tables.h',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/resourceinmemory.cc',
-            #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/resourceinmemory.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_google.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_htmlutils.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_htmlutils_windows.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_logging.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_macros.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_scoped_ptr.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_scopedptr.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_strtoint.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_unicodetext.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_unicodetext.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_unilib.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_unilib_windows.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_utf.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_utf8statetable.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_utf8statetable.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_utf8utils.h',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_utf8utils_windows.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/normalizedunicodetext.cc',
-            'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/normalizedunicodetext.h',
-            'bar/toolbar/cld/i18n/encodings/internal/encodings.cc',
-            'bar/toolbar/cld/i18n/encodings/proto/encodings.pb.h',
-            'bar/toolbar/cld/i18n/encodings/public/encodings.h',
-            'bar/toolbar/cld/i18n/languages/internal/languages.cc',
-            'bar/toolbar/cld/i18n/languages/proto/languages.pb.h',
-            'bar/toolbar/cld/i18n/languages/public/languages.h',
-            'base/basictypes.h',
-            'base/build_config.h',
-            'base/casts.h',
-            'base/commandlineflags.h',
-            'base/global_strip_options.h',
-            'base/logging.h',
-            'base/macros.h',
-            'base/port.h',
-            'base/crash.h',
-            'base/dynamic_annotations.h',
-            'base/scoped_ptr.h',
-            'base/stl_decl_msvc.h',
-            'base/log_severity.h',
-            'base/strtoint.h',
-            'base/vlog_is_on.h',
-            'base/string_util.h',
-            'base/type_traits.h',
-            'base/template_util.h',
-          ],
-          'direct_dependent_settings': {
-            'defines': [
-              'CLD_WINDOWS',
-              'COMPILER_MSVC',
-            ],
-          },
-        },],
+  'targets': [
+    {
+      'target_name': 'cld',
+      'type': '<(library)',
+      'dependencies': [
+	'../icu/icu.gyp:icuuc',
+      ],
+      'include_dirs': [
+        '.',
+      ],
+      'defines': [
+        'CLD_WINDOWS',
+      ],
+      'sources': [
+        'bar/common/scopedptr.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/cldutil.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/cldutil.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/cldutil_dbg.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/cldutil_dbg_empty.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/compact_lang_det.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/compact_lang_det.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/compact_lang_det_impl.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/compact_lang_det_impl.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/ext_lang_enc.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/ext_lang_enc.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/getonescriptspan.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/getonescriptspan.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/letterscript_enum.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/letterscript_enum.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/subsetsequence.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/subsetsequence.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/tote.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/tote.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/utf8propjustletter.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/utf8propletterscriptnum.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/utf8scannotjustletterspecial.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_longwords8_0.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_meanscore.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_quads_128.cc',
+        # For now using the 128 bytes detection in order to save hundreds of KBs on the final package.
+        # 'bar/toolbar/cld/i18n/encodings/compact_lang_det/generated/compact_lang_det_generated_quads_256.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_basictypes.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_commandlineflags.h',
+        # We use the static table at this point, so we don't need to compile the following files:
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_dynamicstate.h',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_dynamicstate.cc',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_loadpolicy.cc',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_loadpolicy.h',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_loadpolicyinterface.h',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_resourceids.h',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_service.h',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_service.cc',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_serviceinterface.h',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_tables.cc',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_tables.h',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/resourceinmemory.cc',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/resourceinmemory.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_google.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_htmlutils.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_htmlutils_windows.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_logging.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_macros.h',
+	# None of files we build require these two headers.
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_scoped_ptr.h',
+        #'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_scopedptr.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_strtoint.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_unicodetext.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_unicodetext.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_unilib.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_unilib_windows.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_utf.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_utf8statetable.cc',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_utf8statetable.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_utf8utils.h',
+        'bar/toolbar/cld/i18n/encodings/compact_lang_det/win/cld_utf8utils_windows.cc',
+        'bar/toolbar/cld/i18n/encodings/internal/encodings.cc',
+        'bar/toolbar/cld/i18n/encodings/proto/encodings.pb.h',
+        'bar/toolbar/cld/i18n/encodings/public/encodings.h',
+        'bar/toolbar/cld/i18n/languages/internal/languages.cc',
+        'bar/toolbar/cld/i18n/languages/proto/languages.pb.h',
+        'bar/toolbar/cld/i18n/languages/public/languages.h',
+        'base/basictypes.h',
+        'base/build_config.h',
+        'base/casts.h',
+        'base/commandlineflags.h',
+        'base/global_strip_options.h',
+        'base/logging.h',
+        'base/macros.h',
+        'base/port.h',
+        'base/crash.h',
+        'base/dynamic_annotations.h',
+        'base/scoped_ptr.h',
+        'base/stl_decl_msvc.h',
+        'base/log_severity.h',
+        'base/strtoint.h',
+        'base/vlog_is_on.h',
+        'base/string_util.h',
+        'base/type_traits.h',
+        'base/template_util.h',
+      ],
+      'direct_dependent_settings': {
+        'defines': [
+          'CLD_WINDOWS',
+        ],
       },
-    ],
+      'conditions': [
+        ['OS=="win"', {
+              'direct_dependent_settings': {
+                'defines': [
+                  'COMPILER_MSVC',
+                ],
+              },
+              'msvs_disabled_warnings': [4005, 4006, 4018, 4244, 4309, 4800],
+            },
+        ],
+        ['OS!="win"', {
+              'direct_dependent_settings': {
+                'defines': [
+                  'COMPILER_GCC',
+                ],
+              },
+            },
+        ],
+      ],
+    },
   ],
 }