Add webkit to the repository.

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@18 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 366 insertions, 0 deletions

diff --git a/webkit/pending/TextCodecICU.cpp b/webkit/pending/TextCodecICU.cpp
new file mode 100644
index 0000000..393deaf
--- /dev/null
+++ b/webkit/pending/TextCodecICU.cpp
@@ -0,0 +1,366 @@
+/*
+ * Copyright (C) 2004, 2006, 2007, 2008 Apple Inc. All rights reserved.
+ * Copyright (C) 2006 Alexey Proskuryakov <ap@nypop.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+ */
+
+#include "config.h"
+#include "TextCodecICU.h"
+
+#include "CharacterNames.h"
+#include "CString.h"
+#include "PlatformString.h"
+#include <unicode/ucnv.h>
+#include <unicode/ucnv_cb.h>
+#include <wtf/Assertions.h>
+#include <wtf/HashMap.h>
+
+using std::auto_ptr;
+using std::min;
+
+namespace WebCore {
+
+const size_t ConversionBufferSize = 16384;
+    
+static UConverter* cachedConverterICU;
+
+static auto_ptr<TextCodec> newTextCodecICU(const TextEncoding& encoding, const void*)
+{
+    return auto_ptr<TextCodec>(new TextCodecICU(encoding));
+}
+
+void TextCodecICU::registerBaseEncodingNames(EncodingNameRegistrar registrar)
+{
+    registrar("UTF-8", "UTF-8");
+}
+
+void TextCodecICU::registerBaseCodecs(TextCodecRegistrar registrar)
+{
+    registrar("UTF-8", newTextCodecICU, 0);
+}
+
+// FIXME: Registering all the encodings we get from ucnv_getAvailableName
+// includes encodings we don't want or need. For example: UTF16_PlatformEndian,
+// UTF16_OppositeEndian, UTF32_PlatformEndian, UTF32_OppositeEndian, and all
+// the encodings with commas and version numbers.
+
+void TextCodecICU::registerExtendedEncodingNames(EncodingNameRegistrar registrar)
+{
+    // We register Hebrew with logical ordering using a separate name.
+    // Otherwise, this would share the same canonical name as the
+    // visual ordering case, and then TextEncoding could not tell them
+    // apart; ICU works with either name.
+    registrar("ISO-8859-8-I", "ISO-8859-8-I");
+
+    int32_t numEncodings = ucnv_countAvailable();
+    for (int32_t i = 0; i < numEncodings; ++i) {
+        const char* name = ucnv_getAvailableName(i);
+        UErrorCode error = U_ZERO_ERROR;
+        // Try MIME before trying IANA to pick up commonly used names like
+        // 'EUC-JP' instead of horrendeously long names like 
+        // 'Extended_UNIX_Code_Packed_Format_for_Japanese'. 
+        const char* standardName = ucnv_getStandardName(name, "MIME", &error);
+        if (!U_SUCCESS(error) || !standardName) {
+            error = U_ZERO_ERROR;
+            // Try IANA to pick up 'windows-12xx' and other names
+            // which are not preferred MIME names but are widely used. 
+            standardName = ucnv_getStandardName(name, "IANA", &error);
+            if (!U_SUCCESS(error) || !standardName)
+                continue;
+        }
+
+        // Here, we used to alias GB2312 and GB2312-80 to GBK, but our copy
+        // of ICU treats GB2312/GB2312-80 as synonyms of GBK so that we
+        // don't need that any more.
+        registrar(standardName, standardName);
+
+        uint16_t numAliases = ucnv_countAliases(name, &error);
+        ASSERT(U_SUCCESS(error));
+        if (U_SUCCESS(error))
+            for (uint16_t j = 0; j < numAliases; ++j) {
+                error = U_ZERO_ERROR;
+                const char* alias = ucnv_getAlias(name, j, &error);
+                ASSERT(U_SUCCESS(error));
+                if (U_SUCCESS(error) && alias != standardName)
+                    registrar(alias, standardName);
+            }
+    }
+
+    // We used to map macroman and xmacroman to macintosh, but 
+    // we don't need them any more because they're added to our
+    // local copy of ICU.
+
+    // Additional aliases that historically were present in the encoding
+    // table in WebKit on Macintosh that don't seem to be present in ICU.
+    // Perhaps we can prove these are not used on the web and remove them.
+    // Or perhaps we can get them added to ICU.
+    registrar("cnbig5", "Big5");
+    registrar("cngb", "EUC-CN");
+    registrar("csISO88598I", "ISO_8859-8-I");
+    registrar("csgb231280", "EUC-CN");
+    registrar("dos720", "cp864");
+    registrar("dos874", "TIS-620");
+    registrar("jis7", "ISO-2022-JP");
+    registrar("koi", "KOI8-R");
+    registrar("logical", "ISO-8859-8-I");
+    registrar("unicode11utf8", "UTF-8");
+    registrar("unicode20utf8", "UTF-8");
+    registrar("visual", "ISO-8859-8");
+    registrar("winarabic", "windows-1256");
+    registrar("winbaltic", "windows-1257");
+    registrar("wincyrillic", "windows-1251");
+    registrar("windows874", "TIS-620");
+    registrar("wingreek", "windows-1253");
+    registrar("winhebrew", "windows-1255");
+    registrar("winlatin2", "windows-1250");
+    registrar("winturkish", "windows-1254");
+    registrar("winvietnamese", "windows-1258");
+    registrar("xcp1250", "windows-1250");
+    registrar("xcp1251", "windows-1251");
+    registrar("xeuc", "EUC-JP");
+    registrar("xeuccn", "EUC-CN");
+    registrar("xgbk", "EUC-CN");
+    registrar("xunicode20utf8", "UTF-8");
+    registrar("xxbig5", "Big5");
+}
+
+void TextCodecICU::registerExtendedCodecs(TextCodecRegistrar registrar)
+{
+    // See comment above in registerEncodingNames.
+    registrar("ISO-8859-8-I", newTextCodecICU, 0);
+
+    int32_t numEncodings = ucnv_countAvailable();
+    for (int32_t i = 0; i < numEncodings; ++i) {
+        const char* name = ucnv_getAvailableName(i);
+        UErrorCode error = U_ZERO_ERROR;
+        // Try MIME before trying IANA to pick up commonly used names like
+        // 'EUC-JP' instead of horrendeously long names like 
+        // 'Extended_UNIX_Code_Packed_Format_for_Japanese'. 
+        const char* standardName = ucnv_getStandardName(name, "MIME", &error);
+        if (!U_SUCCESS(error) || !standardName) {
+            error = U_ZERO_ERROR;
+            // Try IANA to pick up 'windows-12xx' and other names
+            // which are not preferred MIME names but are widely used. 
+            standardName = ucnv_getStandardName(name, "IANA", &error);
+            if (!U_SUCCESS(error) || !standardName)
+                continue;
+        }
+        registrar(standardName, newTextCodecICU, 0);
+    }
+}
+
+TextCodecICU::TextCodecICU(const TextEncoding& encoding)
+    : m_encoding(encoding)
+    , m_numBufferedBytes(0)
+    , m_converterICU(0)
+    , m_needsGBKFallbacks(false)
+{
+}
+
+TextCodecICU::~TextCodecICU()
+{
+    releaseICUConverter();
+}
+
+void TextCodecICU::releaseICUConverter() const
+{
+    if (m_converterICU) {
+        if (cachedConverterICU)
+            ucnv_close(cachedConverterICU);
+        cachedConverterICU = m_converterICU;
+        m_converterICU = 0;
+    }
+}
+
+void TextCodecICU::createICUConverter() const
+{
+    ASSERT(!m_converterICU);
+
+    const char* name = m_encoding.name();
+    m_needsGBKFallbacks = name[0] == 'G' && name[1] == 'B' && name[2] == 'K' && !name[3];
+
+    UErrorCode err;
+
+    if (cachedConverterICU) {
+        err = U_ZERO_ERROR;
+        const char* cachedName = ucnv_getName(cachedConverterICU, &err);
+        if (U_SUCCESS(err) && m_encoding == cachedName) {
+            m_converterICU = cachedConverterICU;
+            cachedConverterICU = 0;
+            return;
+        }
+    }
+
+    err = U_ZERO_ERROR;
+    m_converterICU = ucnv_open(m_encoding.name(), &err);
+#if !LOG_DISABLED
+    if (err == U_AMBIGUOUS_ALIAS_WARNING)
+        LOG_ERROR("ICU ambiguous alias warning for encoding: %s", m_encoding.name());
+#endif
+    if (m_converterICU)
+        ucnv_setFallback(m_converterICU, TRUE);
+}
+
+String TextCodecICU::decode(const char* bytes, size_t length, bool flush)
+{
+    // Get a converter for the passed-in encoding.
+    if (!m_converterICU) {
+        createICUConverter();
+        ASSERT(m_converterICU);
+        if (!m_converterICU) {
+            LOG_ERROR("error creating ICU encoder even though encoding was in table");
+            return String();
+        }
+    }
+
+    Vector<UChar> result;
+
+    UChar buffer[ConversionBufferSize];
+    const char* source = reinterpret_cast<const char*>(bytes);
+    const char* sourceLimit = source + length;
+    int32_t* offsets = NULL;
+    UErrorCode err;
+
+    do {
+        UChar* target = buffer;
+        const UChar* targetLimit = target + ConversionBufferSize;
+        err = U_ZERO_ERROR;
+        ucnv_toUnicode(m_converterICU, &target, targetLimit, &source, sourceLimit, offsets, flush, &err);
+        int count = target - buffer;
+        result.append(buffer, count);
+    } while (err == U_BUFFER_OVERFLOW_ERROR);
+
+    if (U_FAILURE(err)) {
+        // flush the converter so it can be reused, and not be bothered by this error.
+        do {
+            UChar *target = buffer;
+            const UChar *targetLimit = target + ConversionBufferSize;
+            err = U_ZERO_ERROR;
+            ucnv_toUnicode(m_converterICU, &target, targetLimit, &source, sourceLimit, offsets, true, &err);
+        } while (source < sourceLimit);
+        LOG_ERROR("ICU conversion error");
+        return String();
+    }
+
+    String resultString = String::adopt(result);
+
+    // <http://bugs.webkit.org/show_bug.cgi?id=17014>
+    // Simplified Chinese pages use the code A3A0 to mean "full-width space", but ICU decodes it as U+E5E5.
+    if (m_encoding == "GBK" || m_encoding == "gb18030")
+        resultString.replace(0xE5E5, ideographicSpace);
+
+    return resultString;
+}
+
+// We need to apply these fallbacks ourselves as they are not currently supported by ICU and
+// they were provided by the old TEC encoding path
+// Needed to fix <rdar://problem/4708689>
+static HashMap<UChar32, UChar>& gbkEscapes() {
+    static HashMap<UChar32, UChar> escapes;
+    if (escapes.isEmpty()) {
+        escapes.add(0x01F9, 0xE7C8);
+        escapes.add(0x1E3F, 0xE7C7);
+        escapes.add(0x22EF, 0x2026);
+        escapes.add(0x301C, 0xFF5E);
+    }
+        
+    return escapes;
+}
+
+static void gbkCallbackEscape(const void* context, UConverterFromUnicodeArgs* fromUArgs, const UChar* codeUnits, int32_t length,
+                              UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err) 
+{
+    if (codePoint && gbkEscapes().contains(codePoint)) {
+        UChar outChar = gbkEscapes().get(codePoint);
+        const UChar* source = &outChar;
+        *err = U_ZERO_ERROR;
+        ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err);
+        return;
+    }
+    UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint, reason, err);
+}
+
+static void gbkCallbackSubstitute(const void* context, UConverterFromUnicodeArgs* fromUArgs, const UChar* codeUnits, int32_t length,
+                                  UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err) 
+{
+    if (gbkEscapes().contains(codePoint)) {
+        UChar outChar = gbkEscapes().get(codePoint);
+        const UChar* source = &outChar;
+        *err = U_ZERO_ERROR;
+        ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err);
+        return;
+    }
+    UCNV_FROM_U_CALLBACK_SUBSTITUTE(context, fromUArgs, codeUnits, length, codePoint, reason, err);
+}
+
+CString TextCodecICU::encode(const UChar* characters, size_t length, bool allowEntities)
+{
+    if (!length)
+        return "";
+
+    if (!m_converterICU)
+        createICUConverter();
+    if (!m_converterICU)
+        return CString();
+
+    // FIXME: We should see if there is "force ASCII range" mode in ICU;
+    // until then, we change the backslash into a yen sign.
+    // Encoding will change the yen sign back into a backslash.
+    String copy(characters, length);
+    copy.replace('\\', m_encoding.backslashAsCurrencySymbol());
+
+    const UChar* source = copy.characters();
+    const UChar* sourceLimit = source + copy.length();
+    
+    UErrorCode err = U_ZERO_ERROR;
+
+    if (allowEntities)
+        ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkCallbackEscape : UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_DEC, 0, 0, &err);
+    else {
+        ucnv_setSubstChars(m_converterICU, "?", 1, &err);
+        ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkCallbackSubstitute : UCNV_FROM_U_CALLBACK_SUBSTITUTE, 0, 0, 0, &err);
+    }
+
+    ASSERT(U_SUCCESS(err));
+    if (U_FAILURE(err))
+        return CString();
+
+    Vector<char> result;
+    size_t size = 0;
+    do {
+        char buffer[ConversionBufferSize];
+        char* target = buffer;
+        char* targetLimit = target + ConversionBufferSize;
+        err = U_ZERO_ERROR;
+        ucnv_fromUnicode(m_converterICU, &target, targetLimit, &source, sourceLimit, 0, true, &err);
+        size_t count = target - buffer;
+        result.grow(size + count);
+        memcpy(result.data() + size, buffer, count);
+        size += count;
+    } while (err == U_BUFFER_OVERFLOW_ERROR);
+
+    return CString(result.data(), size);
+}
+
+
+} // namespace WebCore