Add a method that truncates strings to the end point of a valid UTF8 character, leaving the string's size to be less than or equal to a specified byte size.

BUG=43675
TEST=base/string_util_unittest.cc

Patch contributed by to Jerrica Jones (jerrica@chromium.org).

Review URL: http://codereview.chromium.org/2239007

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@48518 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 212 insertions, 0 deletions

diff --git a/base/string_util.cc b/base/string_util.cc
index 0269ba1..e36ae51 100644
--- a/base/string_util.cc
+++ b/base/string_util.cc
@@ -483,6 +483,41 @@ bool TrimString(const std::string& input,
   return TrimStringT(input, trim_chars, TRIM_ALL, output) != TRIM_NONE;
 }
 
+void TruncateUTF8ToByteSize(const std::string& input,
+                            const size_t byte_size,
+                            std::string* output) {
+  if (byte_size > input.length()) {
+    *output = input;
+    return;
+  }
+
+  int32 truncation_length = static_cast<int32>(byte_size);
+  int32 char_index = truncation_length - 1;
+  const char* cstr = input.c_str();
+
+  // Using CBU8, we will move backwards from the truncation point
+  // to the beginning of the string looking for a valid UTF8
+  // character.  Once a full UTF8 character is found, we will
+  // truncate the string to the end of that character.
+  while (char_index >= 0) {
+    int32 prev = char_index;
+    uint32 code_point = 0;
+    CBU8_NEXT(cstr, char_index, truncation_length, code_point);
+    if (!base::IsValidCharacter(code_point) ||
+        !base::IsValidCodepoint(code_point)) {
+      char_index = prev - 1;
+    } else {
+      break;
+    }
+  }
+
+  DCHECK(output != NULL);
+  if (char_index >= 0 )
+    *output = input.substr(0, char_index);
+  else
+    output->clear();
+}
+
 TrimPositions TrimWhitespace(const std::wstring& input,
                              TrimPositions positions,
                              std::wstring* output) {
diff --git a/base/string_util.h b/base/string_util.h
index 28cd26f..cb7e5b5 100644
--- a/base/string_util.h
+++ b/base/string_util.h
@@ -164,6 +164,12 @@ bool TrimString(const std::string& input,
                 const char trim_chars[],
                 std::string* output);
 
+// Truncates a string to the nearest UTF-8 character that will leave
+// the string less than or equal to the specified byte size.
+void TruncateUTF8ToByteSize(const std::string& input,
+                            const size_t byte_size,
+                            std::string* output);
+
 // Trims any whitespace from either end of the input string.  Returns where
 // whitespace was found.
 // The non-wide version has two functions:
diff --git a/base/string_util_unittest.cc b/base/string_util_unittest.cc
index 5d41cb0..a84ad5d 100644
--- a/base/string_util_unittest.cc
+++ b/base/string_util_unittest.cc
@@ -78,6 +78,177 @@ static const struct trim_case_ascii {
   {"\t\rTest String\n", TRIM_ALL, "Test String", TRIM_ALL},
 };
 
+namespace {
+
+// Helper used to test TruncateUTF8ToByteSize.
+bool Truncated(const std::string& input, const size_t byte_size,
+               std::string* output) {
+    size_t prev = input.length();
+    TruncateUTF8ToByteSize(input, byte_size, output);
+    return prev != (*output).length();
+}
+
+}  // namespace
+
+TEST(StringUtilTest, TruncateUTF8ToByteSize) {
+  std::string output;
+
+  // Empty strings and invalid byte_size arguments
+  EXPECT_FALSE(Truncated("", 0, &output));
+  EXPECT_EQ(output, "");
+  EXPECT_TRUE(Truncated("\xe1\x80\xbf", 0, &output));
+  EXPECT_EQ(output, "");
+  EXPECT_FALSE(Truncated("\xe1\x80\xbf", -1, &output));
+  EXPECT_FALSE(Truncated("\xe1\x80\xbf", 4, &output));
+
+  // Testing the truncation of valid UTF8 correctly
+  EXPECT_TRUE(Truncated("abc", 2, &output));
+  EXPECT_EQ(output, "ab");
+  EXPECT_TRUE(Truncated("\xc2\x81\xc2\x81", 2, &output));
+  EXPECT_EQ(output.compare("\xc2\x81"), 0);
+  EXPECT_TRUE(Truncated("\xc2\x81\xc2\x81", 3, &output));
+  EXPECT_EQ(output.compare("\xc2\x81"), 0);
+  EXPECT_FALSE(Truncated("\xc2\x81\xc2\x81", 4, &output));
+  EXPECT_EQ(output.compare("\xc2\x81\xc2\x81"), 0);
+
+  {
+    const char array[] = "\x00\x00\xc2\x81\xc2\x81";
+    const std::string array_string(array, arraysize(array));
+    EXPECT_TRUE(Truncated(array_string, 4, &output));
+    EXPECT_EQ(output.compare(std::string("\x00\x00\xc2\x81", 4)), 0);
+  }
+
+  {
+    const char array[] = "\x00\xc2\x81\xc2\x81";
+    const std::string array_string(array, arraysize(array));
+    EXPECT_TRUE(Truncated(array_string, 4, &output));
+    EXPECT_EQ(output.compare(std::string("\x00\xc2\x81", 3)), 0);
+  }
+
+  // Testing invalid UTF8
+  EXPECT_TRUE(Truncated("\xed\xa0\x80\xed\xbf\xbf", 6, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xed\xa0\x8f", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xed\xbf\xbf", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  // Testing invalid UTF8 mixed with valid UTF8
+  EXPECT_FALSE(Truncated("\xe1\x80\xbf", 3, &output));
+  EXPECT_EQ(output.compare("\xe1\x80\xbf"), 0);
+  EXPECT_FALSE(Truncated("\xf1\x80\xa0\xbf", 4, &output));
+  EXPECT_EQ(output.compare("\xf1\x80\xa0\xbf"), 0);
+  EXPECT_FALSE(Truncated("a\xc2\x81\xe1\x80\xbf\xf1\x80\xa0\xbf",
+              10, &output));
+  EXPECT_EQ(output.compare("a\xc2\x81\xe1\x80\xbf\xf1\x80\xa0\xbf"), 0);
+  EXPECT_TRUE(Truncated("a\xc2\x81\xe1\x80\xbf\xf1""a""\x80\xa0",
+              10, &output));
+  EXPECT_EQ(output.compare("a\xc2\x81\xe1\x80\xbf\xf1""a"), 0);
+  EXPECT_FALSE(Truncated("\xef\xbb\xbf" "abc", 6, &output));
+  EXPECT_EQ(output.compare("\xef\xbb\xbf" "abc"), 0);
+
+  // Overlong sequences
+  EXPECT_TRUE(Truncated("\xc0\x80", 2, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xc1\x80\xc1\x81", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xe0\x80\x80", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xe0\x82\x80", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xe0\x9f\xbf", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf0\x80\x80\x8D", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf0\x80\x82\x91", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf0\x80\xa0\x80", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf0\x8f\xbb\xbf", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf8\x80\x80\x80\xbf", 5, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xfc\x80\x80\x80\xa0\xa5", 6, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  // Beyond U+10FFFF (the upper limit of Unicode codespace)
+  EXPECT_TRUE(Truncated("\xf4\x90\x80\x80", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf8\xa0\xbf\x80\xbf", 5, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xfc\x9c\xbf\x80\xbf\x80", 6, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  // BOMs in UTF-16(BE|LE) and UTF-32(BE|LE)
+  EXPECT_TRUE(Truncated("\xfe\xff", 2, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xff\xfe", 2, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  {
+    const char array[] = "\x00\x00\xfe\xff";
+    const std::string array_string(array, arraysize(array));
+    EXPECT_TRUE(Truncated(array_string, 4, &output));
+    EXPECT_EQ(output.compare(std::string("\x00\x00", 2)), 0);
+  }
+
+  // Variants on the previous test
+  {
+    const char array[] = "\xff\xfe\x00\x00";
+    const std::string array_string(array, 4);
+    EXPECT_FALSE(Truncated(array_string, 4, &output));
+    EXPECT_EQ(output.compare(std::string("\xff\xfe\x00\x00", 4)), 0);
+  }
+  {
+    const char array[] = "\xff\x00\x00\xfe";
+    const std::string array_string(array, arraysize(array));
+    EXPECT_TRUE(Truncated(array_string, 4, &output));
+    EXPECT_EQ(output.compare(std::string("\xff\x00\x00", 3)), 0);
+  }
+
+  // Non-characters : U+xxFFF[EF] where xx is 0x00 through 0x10 and <FDD0,FDEF>
+  EXPECT_TRUE(Truncated("\xef\xbf\xbe", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf0\x8f\xbf\xbe", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf3\xbf\xbf\xbf", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xef\xb7\x90", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xef\xb7\xaf", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  // Strings in legacy encodings that are valid in UTF-8, but
+  // are invalid as UTF-8 in real data.
+  EXPECT_TRUE(Truncated("caf\xe9", 4, &output));
+  EXPECT_EQ(output.compare("caf"), 0);
+  EXPECT_TRUE(Truncated("\xb0\xa1\xb0\xa2", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_FALSE(Truncated("\xa7\x41\xa6\x6e", 4, &output));
+  EXPECT_EQ(output.compare("\xa7\x41\xa6\x6e"), 0);
+  EXPECT_TRUE(Truncated("\xa7\x41\xa6\x6e\xd9\xee\xe4\xee", 7,
+              &output));
+  EXPECT_EQ(output.compare("\xa7\x41\xa6\x6e"), 0);
+
+  // Testing using the same string as input and output.
+  EXPECT_FALSE(Truncated(output, 4, &output));
+  EXPECT_EQ(output.compare("\xa7\x41\xa6\x6e"), 0);
+  EXPECT_TRUE(Truncated(output, 3, &output));
+  EXPECT_EQ(output.compare("\xa7\x41"), 0);
+
+  // "abc" with U+201[CD] in windows-125[0-8]
+  EXPECT_TRUE(Truncated("\x93" "abc\x94", 5, &output));
+  EXPECT_EQ(output.compare("\x93" "abc"), 0);
+
+  // U+0639 U+064E U+0644 U+064E in ISO-8859-6
+  EXPECT_TRUE(Truncated("\xd9\xee\xe4\xee", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  // U+03B3 U+03B5 U+03B9 U+03AC in ISO-8859-7
+  EXPECT_TRUE(Truncated("\xe3\xe5\xe9\xdC", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+}
+
 TEST(StringUtilTest, TrimWhitespace) {
   std::wstring output;  // Allow contents to carry over to next testcase
   for (size_t i = 0; i < arraysize(trim_cases); ++i) {