diff options
| -rw-r--r-- | base/base.gyp | 5 | ||||
| -rw-r--r-- | base/i18n/icu_string_conversions.h | 1 | ||||
| -rw-r--r-- | base/utf_offset_string_conversions.cc | 122 | ||||
| -rw-r--r-- | base/utf_offset_string_conversions.h | 35 | ||||
| -rw-r--r-- | base/utf_offset_string_conversions_unittest.cc | 80 | ||||
| -rw-r--r-- | base/utf_string_conversion_utils.cc | 148 | ||||
| -rw-r--r-- | base/utf_string_conversion_utils.h | 86 | ||||
| -rw-r--r-- | base/utf_string_conversions.cc | 240 | ||||
| -rw-r--r-- | base/utf_string_conversions.h | 39 | ||||
| -rw-r--r-- | base/utf_string_conversions_unittest.cc | 43 | ||||
| -rw-r--r-- | net/base/escape.cc | 4 | ||||
| -rw-r--r-- | net/base/net_util.cc | 2 |
12 files changed, 509 insertions, 296 deletions
diff --git a/base/base.gyp b/base/base.gyp index bcb88f6..b954e3b 100644 --- a/base/base.gyp +++ b/base/base.gyp @@ -332,6 +332,10 @@ 'tracked_objects.cc', 'tracked_objects.h', 'tuple.h', + 'utf_offset_string_conversions.cc', + 'utf_offset_string_conversions.h', + 'utf_string_conversion_utils.cc', + 'utf_string_conversion_utils.h', 'utf_string_conversions.cc', 'utf_string_conversions.h', 'unix_domain_socket_posix.cc', @@ -655,6 +659,7 @@ 'timer_unittest.cc', 'tracked_objects_unittest.cc', 'tuple_unittest.cc', + 'utf_offset_string_conversions_unittest.cc', 'utf_string_conversions_unittest.cc', 'values_unittest.cc', 'version_unittest.cc', diff --git a/base/i18n/icu_string_conversions.h b/base/i18n/icu_string_conversions.h index 7b0c77e..79d7e02 100644 --- a/base/i18n/icu_string_conversions.h +++ b/base/i18n/icu_string_conversions.h @@ -8,7 +8,6 @@ #include <string> #include "base/string16.h" -#include "base/string_piece.h" namespace base { diff --git a/base/utf_offset_string_conversions.cc b/base/utf_offset_string_conversions.cc new file mode 100644 index 0000000..69b572e --- /dev/null +++ b/base/utf_offset_string_conversions.cc @@ -0,0 +1,122 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/utf_offset_string_conversions.h" + +#include "base/string_piece.h" +#include "base/utf_string_conversion_utils.h" + +using base::PrepareForUTF16Or32Output; +using base::ReadUnicodeCharacter; +using base::WriteUnicodeCharacter; + +// Generalized Unicode converter ----------------------------------------------- + +// Converts the given source Unicode character type to the given destination +// Unicode character type as a STL string. The given input buffer and size +// determine the source, and the given output STL string will be replaced by +// the result. +template<typename SRC_CHAR> +bool ConvertUnicode(const SRC_CHAR* src, + size_t src_len, + std::wstring* output, + size_t* offset_for_adjustment) { + size_t output_offset = + (offset_for_adjustment && *offset_for_adjustment < src_len) ? + *offset_for_adjustment : std::wstring::npos; + + // ICU requires 32-bit numbers. + bool success = true; + int32 src_len32 = static_cast<int32>(src_len); + for (int32 i = 0; i < src_len32; i++) { + uint32 code_point; + size_t original_i = i; + size_t chars_written = 0; + if (ReadUnicodeCharacter(src, src_len32, &i, &code_point)) { + chars_written = WriteUnicodeCharacter(code_point, output); + } else { + // TODO(jungshik): consider adding 'Replacement character' (U+FFFD) + // in place of an invalid codepoint. + success = false; + } + if ((output_offset != std::wstring::npos) && + (*offset_for_adjustment > original_i)) { + // NOTE: ReadUnicodeCharacter() adjusts |i| to point _at_ the last + // character read, not after it (so that incrementing it in the loop + // increment will place it at the right location), so we need to account + // for that in determining the amount that was read. + if (*offset_for_adjustment <= static_cast<size_t>(i)) + output_offset = std::wstring::npos; + else + output_offset += chars_written - (i - original_i + 1); + } + } + + if (offset_for_adjustment) + *offset_for_adjustment = output_offset; + return success; +} + +// UTF-8 <-> Wide -------------------------------------------------------------- + +bool UTF8ToWideAndAdjustOffset(const char* src, + size_t src_len, + std::wstring* output, + size_t* offset_for_adjustment) { + PrepareForUTF16Or32Output(src, src_len, output); + return ConvertUnicode(src, src_len, output, offset_for_adjustment); +} + +std::wstring UTF8ToWideAndAdjustOffset(const base::StringPiece& utf8, + size_t* offset_for_adjustment) { + std::wstring ret; + UTF8ToWideAndAdjustOffset(utf8.data(), utf8.length(), &ret, + offset_for_adjustment); + return ret; +} + +// UTF-16 <-> Wide ------------------------------------------------------------- + +#if defined(WCHAR_T_IS_UTF16) + +// When wide == UTF-16, then conversions are a NOP. +bool UTF16ToWideAndAdjustOffset(const char16* src, + size_t src_len, + std::wstring* output, + size_t* offset_for_adjustment) { + output->assign(src, src_len); + if (offset_for_adjustment && (*offset_for_adjustment >= src_len)) + *offset_for_adjustment = std::wstring::npos; + return true; +} + +std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, + size_t* offset_for_adjustment) { + if (offset_for_adjustment && (*offset_for_adjustment >= utf16.length())) + *offset_for_adjustment = std::wstring::npos; + return utf16; +} + +#elif defined(WCHAR_T_IS_UTF32) + +bool UTF16ToWideAndAdjustOffset(const char16* src, + size_t src_len, + std::wstring* output, + size_t* offset_for_adjustment) { + output->clear(); + // Assume that normally we won't have any non-BMP characters so the counts + // will be the same. + output->reserve(src_len); + return ConvertUnicode(src, src_len, output, offset_for_adjustment); +} + +std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, + size_t* offset_for_adjustment) { + std::wstring ret; + UTF16ToWideAndAdjustOffset(utf16.data(), utf16.length(), &ret, + offset_for_adjustment); + return ret; +} + +#endif // defined(WCHAR_T_IS_UTF32) diff --git a/base/utf_offset_string_conversions.h b/base/utf_offset_string_conversions.h new file mode 100644 index 0000000..0a1a682 --- /dev/null +++ b/base/utf_offset_string_conversions.h @@ -0,0 +1,35 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef BASE_UTF_OFFSET_STRING_CONVERSIONS_H_ +#define BASE_UTF_OFFSET_STRING_CONVERSIONS_H_ + +#include <string> + +#include "base/string16.h" + +namespace base { +class StringPiece; +} + +// Like the conversions in utf_string_conversions.h, but also take offsets into +// the source strings, which will be adjusted to point at the same logical place +// in the result strings. If this isn't possible because the offsets point past +// the end of the source strings or into the middle of multibyte sequences, they +// will be set to std::wstring::npos. |offset_for_adjustment| may be NULL. +bool UTF8ToWideAndAdjustOffset(const char* src, + size_t src_len, + std::wstring* output, + size_t* offset_for_adjustment); +std::wstring UTF8ToWideAndAdjustOffset(const base::StringPiece& utf8, + size_t* offset_for_adjustment); + +bool UTF16ToWideAndAdjustOffset(const char16* src, + size_t src_len, + std::wstring* output, + size_t* offset_for_adjustment); +std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, + size_t* offset_for_adjustment); + +#endif // BASE_UTF_OFFSET_STRING_CONVERSIONS_H_ diff --git a/base/utf_offset_string_conversions_unittest.cc b/base/utf_offset_string_conversions_unittest.cc new file mode 100644 index 0000000..00d87d3 --- /dev/null +++ b/base/utf_offset_string_conversions_unittest.cc @@ -0,0 +1,80 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/logging.h" +#include "base/string_piece.h" +#include "base/utf_offset_string_conversions.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace base { + +namespace { + +// Given a null-terminated string of wchar_t with each wchar_t representing +// a UTF-16 code unit, returns a string16 made up of wchar_t's in the input. +// Each wchar_t should be <= 0xFFFF and a non-BMP character (> U+FFFF) +// should be represented as a surrogate pair (two UTF-16 units) +// *even* where wchar_t is 32-bit (Linux and Mac). +// +// This is to help write tests for functions with string16 params until +// the C++ 0x UTF-16 literal is well-supported by compilers. +string16 BuildString16(const wchar_t* s) { +#if defined(WCHAR_T_IS_UTF16) + return string16(s); +#elif defined(WCHAR_T_IS_UTF32) + string16 u16; + while (*s != 0) { + DCHECK(static_cast<unsigned int>(*s) <= 0xFFFFu); + u16.push_back(*s++); + } + return u16; +#endif +} + +} // namespace + +TEST(UTFOffsetStringConversionsTest, AdjustOffset) { + struct UTF8ToWideCase { + const char* utf8; + size_t input_offset; + size_t output_offset; + } utf8_to_wide_cases[] = { + {"", 0, std::wstring::npos}, + {"\xe4\xbd\xa0\xe5\xa5\xbd", 1, std::wstring::npos}, + {"\xe4\xbd\xa0\xe5\xa5\xbd", 3, 1}, + {"\xed\xb0\x80z", 3, 0}, + {"A\xF0\x90\x8C\x80z", 1, 1}, + {"A\xF0\x90\x8C\x80z", 2, std::wstring::npos}, +#if defined(WCHAR_T_IS_UTF16) + {"A\xF0\x90\x8C\x80z", 5, 3}, +#elif defined(WCHAR_T_IS_UTF32) + {"A\xF0\x90\x8C\x80z", 5, 2}, +#endif + }; + for (size_t i = 0; i < ARRAYSIZE_UNSAFE(utf8_to_wide_cases); ++i) { + size_t offset = utf8_to_wide_cases[i].input_offset; + UTF8ToWideAndAdjustOffset(utf8_to_wide_cases[i].utf8, &offset); + EXPECT_EQ(utf8_to_wide_cases[i].output_offset, offset); + } + +#if defined(WCHAR_T_IS_UTF32) + struct UTF16ToWideCase { + const wchar_t* wide; + size_t input_offset; + size_t output_offset; + } utf16_to_wide_cases[] = { + {L"\xD840\xDC00\x4E00", 0, 0}, + {L"\xD840\xDC00\x4E00", 1, std::wstring::npos}, + {L"\xD840\xDC00\x4E00", 2, 1}, + }; + for (size_t i = 0; i < ARRAYSIZE_UNSAFE(utf16_to_wide_cases); ++i) { + size_t offset = utf16_to_wide_cases[i].input_offset; + UTF16ToWideAndAdjustOffset(BuildString16(utf16_to_wide_cases[i].wide), + &offset); + EXPECT_EQ(utf16_to_wide_cases[i].output_offset, offset); + } +#endif +} + +} // namaspace base diff --git a/base/utf_string_conversion_utils.cc b/base/utf_string_conversion_utils.cc new file mode 100644 index 0000000..ebd8fd9 --- /dev/null +++ b/base/utf_string_conversion_utils.cc @@ -0,0 +1,148 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/utf_string_conversion_utils.h" + +#include "base/third_party/icu/icu_utf.h" + +namespace base { + +// ReadUnicodeCharacter -------------------------------------------------------- + +bool ReadUnicodeCharacter(const char* src, + int32 src_len, + int32* char_index, + uint32* code_point_out) { + // U8_NEXT expects to be able to use -1 to signal an error, so we must + // use a signed type for code_point. But this function returns false + // on error anyway, so code_point_out is unsigned. + int32 code_point; + CBU8_NEXT(src, *char_index, src_len, code_point); + *code_point_out = static_cast<uint32>(code_point); + + // The ICU macro above moves to the next char, we want to point to the last + // char consumed. + (*char_index)--; + + // Validate the decoded value. + return IsValidCodepoint(code_point); +} + +bool ReadUnicodeCharacter(const char16* src, + int32 src_len, + int32* char_index, + uint32* code_point) { + if (CBU16_IS_SURROGATE(src[*char_index])) { + if (!CBU16_IS_SURROGATE_LEAD(src[*char_index]) || + *char_index + 1 >= src_len || + !CBU16_IS_TRAIL(src[*char_index + 1])) { + // Invalid surrogate pair. + return false; + } + + // Valid surrogate pair. + *code_point = CBU16_GET_SUPPLEMENTARY(src[*char_index], + src[*char_index + 1]); + (*char_index)++; + } else { + // Not a surrogate, just one 16-bit word. + *code_point = src[*char_index]; + } + + return IsValidCodepoint(*code_point); +} + +#if defined(WCHAR_T_IS_UTF32) +bool ReadUnicodeCharacter(const wchar_t* src, + int32 src_len, + int32* char_index, + uint32* code_point) { + // Conversion is easy since the source is 32-bit. + *code_point = src[*char_index]; + + // Validate the value. + return IsValidCodepoint(*code_point); +} +#endif // defined(WCHAR_T_IS_UTF32) + +// WriteUnicodeCharacter ------------------------------------------------------- + +size_t WriteUnicodeCharacter(uint32 code_point, std::string* output) { + if (code_point <= 0x7f) { + // Fast path the common case of one byte. + output->push_back(code_point); + return 1; + } + + + // CBU8_APPEND_UNSAFE can append up to 4 bytes. + size_t char_offset = output->length(); + size_t original_char_offset = char_offset; + output->resize(char_offset + CBU8_MAX_LENGTH); + + CBU8_APPEND_UNSAFE(&(*output)[0], char_offset, code_point); + + // CBU8_APPEND_UNSAFE will advance our pointer past the inserted character, so + // it will represent the new length of the string. + output->resize(char_offset); + return char_offset - original_char_offset; +} + +size_t WriteUnicodeCharacter(uint32 code_point, string16* output) { + if (CBU16_LENGTH(code_point) == 1) { + // Thie code point is in the Basic Multilingual Plane (BMP). + output->push_back(static_cast<char16>(code_point)); + return 1; + } + // Non-BMP characters use a double-character encoding. + size_t char_offset = output->length(); + output->resize(char_offset + CBU16_MAX_LENGTH); + CBU16_APPEND_UNSAFE(&(*output)[0], char_offset, code_point); + return CBU16_MAX_LENGTH; +} + +// Generalized Unicode converter ----------------------------------------------- + +template<typename CHAR> +void PrepareForUTF8Output(const CHAR* src, + size_t src_len, + std::string* output) { + output->clear(); + if (src_len == 0) + return; + if (src[0] < 0x80) { + // Assume that the entire input will be ASCII. + output->reserve(src_len); + } else { + // Assume that the entire input is non-ASCII and will have 3 bytes per char. + output->reserve(src_len * 3); + } +} + +// Instantiate versions we know callers will need. +template void PrepareForUTF8Output(const wchar_t*, size_t, std::string*); +template void PrepareForUTF8Output(const char16*, size_t, std::string*); + +template<typename STRING> +void PrepareForUTF16Or32Output(const char* src, + size_t src_len, + STRING* output) { + output->clear(); + if (src_len == 0) + return; + if (static_cast<unsigned char>(src[0]) < 0x80) { + // Assume the input is all ASCII, which means 1:1 correspondence. + output->reserve(src_len); + } else { + // Otherwise assume that the UTF-8 sequences will have 2 bytes for each + // character. + output->reserve(src_len / 2); + } +} + +// Instantiate versions we know callers will need. +template void PrepareForUTF16Or32Output(const char*, size_t, std::wstring*); +template void PrepareForUTF16Or32Output(const char*, size_t, string16*); + +} // namespace base diff --git a/base/utf_string_conversion_utils.h b/base/utf_string_conversion_utils.h new file mode 100644 index 0000000..a8a76c5 --- /dev/null +++ b/base/utf_string_conversion_utils.h @@ -0,0 +1,86 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef BASE_UTF_STRING_CONVERSION_UTILS_H_ +#define BASE_UTF_STRING_CONVERSION_UTILS_H_ + +// This should only be used by the various UTF string conversion files. + +#include "base/string16.h" + +namespace base { + +inline bool IsValidCodepoint(uint32 code_point) { + // Excludes the surrogate code points ([0xD800, 0xDFFF]) and + // codepoints larger than 0x10FFFF (the highest codepoint allowed). + // Non-characters and unassigned codepoints are allowed. + return code_point < 0xD800u || + (code_point >= 0xE000u && code_point <= 0x10FFFFu); +} + +// ReadUnicodeCharacter -------------------------------------------------------- + +// Reads a UTF-8 stream, placing the next code point into the given output +// |*code_point|. |src| represents the entire string to read, and |*char_index| +// is the character offset within the string to start reading at. |*char_index| +// will be updated to index the last character read, such that incrementing it +// (as in a for loop) will take the reader to the next character. +// +// Returns true on success. On false, |*code_point| will be invalid. +bool ReadUnicodeCharacter(const char* src, + int32 src_len, + int32* char_index, + uint32* code_point_out); + +// Reads a UTF-16 character. The usage is the same as the 8-bit version above. +bool ReadUnicodeCharacter(const char16* src, + int32 src_len, + int32* char_index, + uint32* code_point); + +#if defined(WCHAR_T_IS_UTF32) +// Reads UTF-32 character. The usage is the same as the 8-bit version above. +bool ReadUnicodeCharacter(const wchar_t* src, + int32 src_len, + int32* char_index, + uint32* code_point); +#endif // defined(WCHAR_T_IS_UTF32) + +// WriteUnicodeCharacter ------------------------------------------------------- + +// Appends a UTF-8 character to the given 8-bit string. Returns the number of +// bytes written. +size_t WriteUnicodeCharacter(uint32 code_point, std::string* output); + +// Appends the given code point as a UTF-16 character to the given 16-bit +// string. Returns the number of 16-bit values written. +size_t WriteUnicodeCharacter(uint32 code_point, string16* output); + +#if defined(WCHAR_T_IS_UTF32) +// Appends the given UTF-32 character to the given 32-bit string. Returns the +// number of 32-bit values written. +inline size_t WriteUnicodeCharacter(uint32 code_point, std::wstring* output) { + // This is the easy case, just append the character. + output->push_back(code_point); + return 1; +} +#endif // defined(WCHAR_T_IS_UTF32) + +// Generalized Unicode converter ----------------------------------------------- + +// Guesses the length of the output in UTF-8 in bytes, clears that output +// string, and reserves that amount of space. We assume that the input +// character types are unsigned, which will be true for UTF-16 and -32 on our +// systems. +template<typename CHAR> +void PrepareForUTF8Output(const CHAR* src, size_t src_len, std::string* output); + +// Prepares an output buffer (containing either UTF-16 or -32 data) given some +// UTF-8 input that will be converted to it. See PrepareForUTF8Output(). +template<typename STRING> +void PrepareForUTF16Or32Output(const char* src, size_t src_len, STRING* output); + +} // namespace base + +#endif // BASE_UTF_STRING_CONVERSION_UTILS_H_ diff --git a/base/utf_string_conversions.cc b/base/utf_string_conversions.cc index ee52f47..7376933 100644 --- a/base/utf_string_conversions.cc +++ b/base/utf_string_conversions.cc @@ -4,133 +4,16 @@ #include "base/utf_string_conversions.h" -#include <vector> +#include "base/string_piece.h" +#include "base/utf_string_conversion_utils.h" -#include "base/basictypes.h" -#include "base/logging.h" -#include "base/string_util.h" -#include "base/third_party/icu/icu_utf.h" +using base::PrepareForUTF8Output; +using base::PrepareForUTF16Or32Output; +using base::ReadUnicodeCharacter; +using base::WriteUnicodeCharacter; namespace { -inline bool IsValidCodepoint(uint32 code_point) { - // Excludes the surrogate code points ([0xD800, 0xDFFF]) and - // codepoints larger than 0x10FFFF (the highest codepoint allowed). - // Non-characters and unassigned codepoints are allowed. - return code_point < 0xD800u || - (code_point >= 0xE000u && code_point <= 0x10FFFFu); -} - -// ReadUnicodeCharacter -------------------------------------------------------- - -// Reads a UTF-8 stream, placing the next code point into the given output -// |*code_point|. |src| represents the entire string to read, and |*char_index| -// is the character offset within the string to start reading at. |*char_index| -// will be updated to index the last character read, such that incrementing it -// (as in a for loop) will take the reader to the next character. -// -// Returns true on success. On false, |*code_point| will be invalid. -bool ReadUnicodeCharacter(const char* src, int32 src_len, - int32* char_index, uint32* code_point_out) { - // U8_NEXT expects to be able to use -1 to signal an error, so we must - // use a signed type for code_point. But this function returns false - // on error anyway, so code_point_out is unsigned. - int32 code_point; - CBU8_NEXT(src, *char_index, src_len, code_point); - *code_point_out = static_cast<uint32>(code_point); - - // The ICU macro above moves to the next char, we want to point to the last - // char consumed. - (*char_index)--; - - // Validate the decoded value. - return IsValidCodepoint(code_point); -} - -// Reads a UTF-16 character. The usage is the same as the 8-bit version above. -bool ReadUnicodeCharacter(const char16* src, int32 src_len, - int32* char_index, uint32* code_point) { - if (CBU16_IS_SURROGATE(src[*char_index])) { - if (!CBU16_IS_SURROGATE_LEAD(src[*char_index]) || - *char_index + 1 >= src_len || - !CBU16_IS_TRAIL(src[*char_index + 1])) { - // Invalid surrogate pair. - return false; - } - - // Valid surrogate pair. - *code_point = CBU16_GET_SUPPLEMENTARY(src[*char_index], - src[*char_index + 1]); - (*char_index)++; - } else { - // Not a surrogate, just one 16-bit word. - *code_point = src[*char_index]; - } - - return IsValidCodepoint(*code_point); -} - -#if defined(WCHAR_T_IS_UTF32) -// Reads UTF-32 character. The usage is the same as the 8-bit version above. -bool ReadUnicodeCharacter(const wchar_t* src, int32 src_len, - int32* char_index, uint32* code_point) { - // Conversion is easy since the source is 32-bit. - *code_point = src[*char_index]; - - // Validate the value. - return IsValidCodepoint(*code_point); -} -#endif // defined(WCHAR_T_IS_UTF32) - -// WriteUnicodeCharacter ------------------------------------------------------- - -// Appends a UTF-8 character to the given 8-bit string. Returns the number of -// bytes written. -size_t WriteUnicodeCharacter(uint32 code_point, std::string* output) { - if (code_point <= 0x7f) { - // Fast path the common case of one byte. - output->push_back(code_point); - return 1; - } - - // CBU8_APPEND_UNSAFE can append up to 4 bytes. - size_t char_offset = output->length(); - size_t original_char_offset = char_offset; - output->resize(char_offset + CBU8_MAX_LENGTH); - - CBU8_APPEND_UNSAFE(&(*output)[0], char_offset, code_point); - - // CBU8_APPEND_UNSAFE will advance our pointer past the inserted character, so - // it will represent the new length of the string. - output->resize(char_offset); - return char_offset - original_char_offset; -} - -// Appends the given code point as a UTF-16 character to the given 16-bit -// string. Returns the number of 16-bit values written. -size_t WriteUnicodeCharacter(uint32 code_point, string16* output) { - if (CBU16_LENGTH(code_point) == 1) { - // Thie code point is in the Basic Multilingual Plane (BMP). - output->push_back(static_cast<char16>(code_point)); - return 1; - } - // Non-BMP characters use a double-character encoding. - size_t char_offset = output->length(); - output->resize(char_offset + CBU16_MAX_LENGTH); - CBU16_APPEND_UNSAFE(&(*output)[0], char_offset, code_point); - return CBU16_MAX_LENGTH; -} - -#if defined(WCHAR_T_IS_UTF32) -// Appends the given UTF-32 character to the given 32-bit string. Returns the -// number of 32-bit values written. -inline size_t WriteUnicodeCharacter(uint32 code_point, std::wstring* output) { - // This is the easy case, just append the character. - output->push_back(code_point); - return 1; -} -#endif // defined(WCHAR_T_IS_UTF32) - // Generalized Unicode converter ----------------------------------------------- // Converts the given source Unicode character type to the given destination @@ -140,90 +23,31 @@ inline size_t WriteUnicodeCharacter(uint32 code_point, std::wstring* output) { template<typename SRC_CHAR, typename DEST_STRING> bool ConvertUnicode(const SRC_CHAR* src, size_t src_len, - DEST_STRING* output, - size_t* offset_for_adjustment) { - size_t output_offset = - (offset_for_adjustment && *offset_for_adjustment < src_len) ? - *offset_for_adjustment : DEST_STRING::npos; - + DEST_STRING* output) { // ICU requires 32-bit numbers. bool success = true; int32 src_len32 = static_cast<int32>(src_len); for (int32 i = 0; i < src_len32; i++) { uint32 code_point; - size_t original_i = i; - size_t chars_written = 0; if (ReadUnicodeCharacter(src, src_len32, &i, &code_point)) { - chars_written = WriteUnicodeCharacter(code_point, output); + WriteUnicodeCharacter(code_point, output); } else { // TODO(jungshik): consider adding 'Replacement character' (U+FFFD) // in place of an invalid codepoint. success = false; } - if ((output_offset != DEST_STRING::npos) && - (*offset_for_adjustment > original_i)) { - // NOTE: ReadUnicodeCharacter() adjusts |i| to point _at_ the last - // character read, not after it (so that incrementing it in the loop - // increment will place it at the right location), so we need to account - // for that in determining the amount that was read. - if (*offset_for_adjustment <= static_cast<size_t>(i)) - output_offset = DEST_STRING::npos; - else - output_offset += chars_written - (i - original_i + 1); - } } - if (offset_for_adjustment) - *offset_for_adjustment = output_offset; return success; } -// Guesses the length of the output in UTF-8 in bytes, clears that output -// string, and reserves that amount of space. We assume that the input -// character types are unsigned, which will be true for UTF-16 and -32 on our -// systems. -template<typename CHAR> -void PrepareForUTF8Output(const CHAR* src, - size_t src_len, - std::string* output) { - output->clear(); - if (src_len == 0) - return; - if (src[0] < 0x80) { - // Assume that the entire input will be ASCII. - output->reserve(src_len); - } else { - // Assume that the entire input is non-ASCII and will have 3 bytes per char. - output->reserve(src_len * 3); - } -} - -// Prepares an output buffer (containing either UTF-16 or -32 data) given some -// UTF-8 input that will be converted to it. See PrepareForUTF8Output(). -template<typename STRING> -void PrepareForUTF16Or32Output(const char* src, - size_t src_len, - STRING* output) { - output->clear(); - if (src_len == 0) - return; - if (static_cast<unsigned char>(src[0]) < 0x80) { - // Assume the input is all ASCII, which means 1:1 correspondence. - output->reserve(src_len); - } else { - // Otherwise assume that the UTF-8 sequences will have 2 bytes for each - // character. - output->reserve(src_len / 2); - } -} - } // namespace // UTF-8 <-> Wide -------------------------------------------------------------- bool WideToUTF8(const wchar_t* src, size_t src_len, std::string* output) { PrepareForUTF8Output(src, src_len, output); - return ConvertUnicode<wchar_t, std::string>(src, src_len, output, NULL); + return ConvertUnicode(src, src_len, output); } std::string WideToUTF8(const std::wstring& wide) { @@ -234,20 +58,14 @@ std::string WideToUTF8(const std::wstring& wide) { return ret; } -bool UTF8ToWideAndAdjustOffset(const char* src, - size_t src_len, - std::wstring* output, - size_t* offset_for_adjustment) { +bool UTF8ToWide(const char* src, size_t src_len, std::wstring* output) { PrepareForUTF16Or32Output(src, src_len, output); - return ConvertUnicode<char, std::wstring>(src, src_len, output, - offset_for_adjustment); + return ConvertUnicode(src, src_len, output); } -std::wstring UTF8ToWideAndAdjustOffset(const base::StringPiece& utf8, - size_t* offset_for_adjustment) { +std::wstring UTF8ToWide(const base::StringPiece& utf8) { std::wstring ret; - UTF8ToWideAndAdjustOffset(utf8.data(), utf8.length(), &ret, - offset_for_adjustment); + UTF8ToWide(utf8.data(), utf8.length(), &ret); return ret; } @@ -265,20 +83,12 @@ string16 WideToUTF16(const std::wstring& wide) { return wide; } -bool UTF16ToWideAndAdjustOffset(const char16* src, - size_t src_len, - std::wstring* output, - size_t* offset_for_adjustment) { +bool UTF16ToWide(const char16* src, size_t src_len, std::wstring* output) { output->assign(src, src_len); - if (offset_for_adjustment && (*offset_for_adjustment >= src_len)) - *offset_for_adjustment = std::wstring::npos; return true; } -std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, - size_t* offset_for_adjustment) { - if (offset_for_adjustment && (*offset_for_adjustment >= utf16.length())) - *offset_for_adjustment = std::wstring::npos; +std::wstring UTF16ToWide(const string16& utf16) { return utf16; } @@ -289,7 +99,7 @@ bool WideToUTF16(const wchar_t* src, size_t src_len, string16* output) { // Assume that normally we won't have any non-BMP characters so the counts // will be the same. output->reserve(src_len); - return ConvertUnicode<wchar_t, string16>(src, src_len, output, NULL); + return ConvertUnicode(src, src_len, output); } string16 WideToUTF16(const std::wstring& wide) { @@ -298,23 +108,17 @@ string16 WideToUTF16(const std::wstring& wide) { return ret; } -bool UTF16ToWideAndAdjustOffset(const char16* src, - size_t src_len, - std::wstring* output, - size_t* offset_for_adjustment) { +bool UTF16ToWide(const char16* src, size_t src_len, std::wstring* output) { output->clear(); // Assume that normally we won't have any non-BMP characters so the counts // will be the same. output->reserve(src_len); - return ConvertUnicode<char16, std::wstring>(src, src_len, output, - offset_for_adjustment); + return ConvertUnicode(src, src_len, output); } -std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, - size_t* offset_for_adjustment) { +std::wstring UTF16ToWide(const string16& utf16) { std::wstring ret; - UTF16ToWideAndAdjustOffset(utf16.data(), utf16.length(), &ret, - offset_for_adjustment); + UTF16ToWide(utf16.data(), utf16.length(), &ret); return ret; } @@ -326,7 +130,7 @@ std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, bool UTF8ToUTF16(const char* src, size_t src_len, string16* output) { PrepareForUTF16Or32Output(src, src_len, output); - return ConvertUnicode<char, string16>(src, src_len, output, NULL); + return ConvertUnicode(src, src_len, output); } string16 UTF8ToUTF16(const std::string& utf8) { @@ -339,7 +143,7 @@ string16 UTF8ToUTF16(const std::string& utf8) { bool UTF16ToUTF8(const char16* src, size_t src_len, std::string* output) { PrepareForUTF8Output(src, src_len, output); - return ConvertUnicode<char16, std::string>(src, src_len, output, NULL); + return ConvertUnicode(src, src_len, output); } std::string UTF16ToUTF8(const string16& utf16) { diff --git a/base/utf_string_conversions.h b/base/utf_string_conversions.h index 7069f83..71e2cb2 100644 --- a/base/utf_string_conversions.h +++ b/base/utf_string_conversions.h @@ -8,26 +8,10 @@ #include <string> #include "base/string16.h" -#include "base/string_piece.h" -// Like the conversions below, but also takes an offset into the source string, -// which will be adjusted to point at the same logical place in the result -// string. If this isn't possible because it points past the end of the source -// string or into the middle of a multibyte sequence, it will be set to -// std::wstring::npos. |offset_for_adjustment| may be NULL. -bool UTF8ToWideAndAdjustOffset(const char* src, - size_t src_len, - std::wstring* output, - size_t* offset_for_adjustment); -std::wstring UTF8ToWideAndAdjustOffset(const base::StringPiece& utf8, - size_t* offset_for_adjustment); - -bool UTF16ToWideAndAdjustOffset(const char16* src, - size_t src_len, - std::wstring* output, - size_t* offset_for_adjustment); -std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, - size_t* offset_for_adjustment); +namespace base { +class StringPiece; +} // These convert between UTF-8, -16, and -32 strings. They are potentially slow, // so avoid unnecessary conversions. The low-level versions return a boolean @@ -44,22 +28,13 @@ std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16, // some situations. bool WideToUTF8(const wchar_t* src, size_t src_len, std::string* output); std::string WideToUTF8(const std::wstring& wide); -inline bool UTF8ToWide(const char* src, size_t src_len, std::wstring* output) { - return UTF8ToWideAndAdjustOffset(src, src_len, output, NULL); -} -inline std::wstring UTF8ToWide(const base::StringPiece& utf8) { - return UTF8ToWideAndAdjustOffset(utf8, NULL); -} +bool UTF8ToWide(const char* src, size_t src_len, std::wstring* output); +std::wstring UTF8ToWide(const base::StringPiece& utf8); bool WideToUTF16(const wchar_t* src, size_t src_len, string16* output); string16 WideToUTF16(const std::wstring& wide); -inline bool UTF16ToWide(const char16* src, size_t src_len, - std::wstring* output) { - return UTF16ToWideAndAdjustOffset(src, src_len, output, NULL); -} -inline std::wstring UTF16ToWide(const string16& utf16) { - return UTF16ToWideAndAdjustOffset(utf16, NULL); -} +bool UTF16ToWide(const char16* src, size_t src_len, std::wstring* output); +std::wstring UTF16ToWide(const string16& utf16); bool UTF8ToUTF16(const char* src, size_t src_len, string16* output); string16 UTF8ToUTF16(const std::string& utf8); diff --git a/base/utf_string_conversions_unittest.cc b/base/utf_string_conversions_unittest.cc index ca79ec5..19189971 100644 --- a/base/utf_string_conversions_unittest.cc +++ b/base/utf_string_conversions_unittest.cc @@ -226,47 +226,4 @@ TEST(UTFStringConversionsTest, ConvertMultiString) { EXPECT_EQ(expected, converted); } -TEST(UTFStringConversionsTest, AdjustOffset) { - struct UTF8ToWideCase { - const char* utf8; - size_t input_offset; - size_t output_offset; - } utf8_to_wide_cases[] = { - {"", 0, std::wstring::npos}, - {"\xe4\xbd\xa0\xe5\xa5\xbd", 1, std::wstring::npos}, - {"\xe4\xbd\xa0\xe5\xa5\xbd", 3, 1}, - {"\xed\xb0\x80z", 3, 0}, - {"A\xF0\x90\x8C\x80z", 1, 1}, - {"A\xF0\x90\x8C\x80z", 2, std::wstring::npos}, -#if defined(WCHAR_T_IS_UTF16) - {"A\xF0\x90\x8C\x80z", 5, 3}, -#elif defined(WCHAR_T_IS_UTF32) - {"A\xF0\x90\x8C\x80z", 5, 2}, -#endif - }; - for (size_t i = 0; i < ARRAYSIZE_UNSAFE(utf8_to_wide_cases); ++i) { - size_t offset = utf8_to_wide_cases[i].input_offset; - UTF8ToWideAndAdjustOffset(utf8_to_wide_cases[i].utf8, &offset); - EXPECT_EQ(utf8_to_wide_cases[i].output_offset, offset); - } - -#if defined(WCHAR_T_IS_UTF32) - struct UTF16ToWideCase { - const wchar_t* wide; - size_t input_offset; - size_t output_offset; - } utf16_to_wide_cases[] = { - {L"\xD840\xDC00\x4E00", 0, 0}, - {L"\xD840\xDC00\x4E00", 1, std::wstring::npos}, - {L"\xD840\xDC00\x4E00", 2, 1}, - }; - for (size_t i = 0; i < ARRAYSIZE_UNSAFE(utf16_to_wide_cases); ++i) { - size_t offset = utf16_to_wide_cases[i].input_offset; - UTF16ToWideAndAdjustOffset(BuildString16(utf16_to_wide_cases[i].wide), - &offset); - EXPECT_EQ(utf16_to_wide_cases[i].output_offset, offset); - } -#endif -} - } // namaspace base diff --git a/net/base/escape.cc b/net/base/escape.cc index 0327fc2..6963967 100644 --- a/net/base/escape.cc +++ b/net/base/escape.cc @@ -8,7 +8,9 @@ #include "base/i18n/icu_string_conversions.h" #include "base/logging.h" -#include "base/string_util.h" +#include "base/string_piece.h" +#include "base/utf_string_conversions.h" +#include "base/utf_offset_string_conversions.h" namespace { diff --git a/net/base/net_util.cc b/net/base/net_util.cc index 309fda7..af090bd 100644 --- a/net/base/net_util.cc +++ b/net/base/net_util.cc @@ -44,7 +44,7 @@ #include "base/string_util.h" #include "base/sys_string_conversions.h" #include "base/time.h" -#include "base/utf_string_conversions.h" +#include "base/utf_offset_string_conversions.h" #include "grit/net_resources.h" #include "googleurl/src/gurl.h" #include "googleurl/src/url_canon.h" |