Add Tokenize() for StringPiece.

BUG=none
TEST=StringUtilTest.TokenizeStringPiece

Review URL: http://codereview.chromium.org/2463007

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@48792 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 59 insertions, 41 deletions

diff --git a/base/string_util.cc b/base/string_util.cc
index e36ae51..216b047 100644
--- a/base/string_util.cc
+++ b/base/string_util.cc
@@ -1428,6 +1428,12 @@ size_t Tokenize(const std::string& str,
   return TokenizeT(str, delimiters, tokens);
 }
 
+size_t Tokenize(const base::StringPiece& str,
+                const base::StringPiece& delimiters,
+                std::vector<base::StringPiece>* tokens) {
+  return TokenizeT(str, delimiters, tokens);
+}
+
 template<typename STR>
 static STR JoinStringT(const std::vector<STR>& parts,
                        typename STR::value_type sep) {
diff --git a/base/string_util.h b/base/string_util.h
index cb7e5b5..11a9fd2 100644
--- a/base/string_util.h
+++ b/base/string_util.h
@@ -601,6 +601,9 @@ size_t Tokenize(const string16& str,
 size_t Tokenize(const std::string& str,
                 const std::string& delimiters,
                 std::vector<std::string>* tokens);
+size_t Tokenize(const base::StringPiece& str,
+                const base::StringPiece& delimiters,
+                std::vector<base::StringPiece>* tokens);
 
 // Does the opposite of SplitString().
 std::wstring JoinString(const std::vector<std::wstring>& parts, wchar_t s);
diff --git a/base/string_util_unittest.cc b/base/string_util_unittest.cc
index a84ad5d..2fdd62e 100644
--- a/base/string_util_unittest.cc
+++ b/base/string_util_unittest.cc
@@ -1247,98 +1247,107 @@ TEST(StringUtilTest, SplitString) {
 }
 
 // Test for Tokenize
-TEST(StringUtilTest, Tokenize) {
-  std::vector<std::string> r;
+template <typename STR>
+void TokenizeTest() {
+  std::vector<STR> r;
   size_t size;
 
-  size = Tokenize("This is a string", " ", &r);
+  size = Tokenize(STR("This is a string"), STR(" "), &r);
   EXPECT_EQ(4U, size);
   ASSERT_EQ(4U, r.size());
-  EXPECT_EQ(r[0], "This");
-  EXPECT_EQ(r[1], "is");
-  EXPECT_EQ(r[2], "a");
-  EXPECT_EQ(r[3], "string");
+  EXPECT_EQ(r[0], STR("This"));
+  EXPECT_EQ(r[1], STR("is"));
+  EXPECT_EQ(r[2], STR("a"));
+  EXPECT_EQ(r[3], STR("string"));
   r.clear();
 
-  size = Tokenize("one,two,three", ",", &r);
+  size = Tokenize(STR("one,two,three"), STR(","), &r);
   EXPECT_EQ(3U, size);
   ASSERT_EQ(3U, r.size());
-  EXPECT_EQ(r[0], "one");
-  EXPECT_EQ(r[1], "two");
-  EXPECT_EQ(r[2], "three");
+  EXPECT_EQ(r[0], STR("one"));
+  EXPECT_EQ(r[1], STR("two"));
+  EXPECT_EQ(r[2], STR("three"));
   r.clear();
 
-  size = Tokenize("one,two:three;four", ",:", &r);
+  size = Tokenize(STR("one,two:three;four"), STR(",:"), &r);
   EXPECT_EQ(3U, size);
   ASSERT_EQ(3U, r.size());
-  EXPECT_EQ(r[0], "one");
-  EXPECT_EQ(r[1], "two");
-  EXPECT_EQ(r[2], "three;four");
+  EXPECT_EQ(r[0], STR("one"));
+  EXPECT_EQ(r[1], STR("two"));
+  EXPECT_EQ(r[2], STR("three;four"));
   r.clear();
 
-  size = Tokenize("one,two:three;four", ";,:", &r);
+  size = Tokenize(STR("one,two:three;four"), STR(";,:"), &r);
   EXPECT_EQ(4U, size);
   ASSERT_EQ(4U, r.size());
-  EXPECT_EQ(r[0], "one");
-  EXPECT_EQ(r[1], "two");
-  EXPECT_EQ(r[2], "three");
-  EXPECT_EQ(r[3], "four");
+  EXPECT_EQ(r[0], STR("one"));
+  EXPECT_EQ(r[1], STR("two"));
+  EXPECT_EQ(r[2], STR("three"));
+  EXPECT_EQ(r[3], STR("four"));
   r.clear();
 
-  size = Tokenize("one, two, three", ",", &r);
+  size = Tokenize(STR("one, two, three"), STR(","), &r);
   EXPECT_EQ(3U, size);
   ASSERT_EQ(3U, r.size());
-  EXPECT_EQ(r[0], "one");
-  EXPECT_EQ(r[1], " two");
-  EXPECT_EQ(r[2], " three");
+  EXPECT_EQ(r[0], STR("one"));
+  EXPECT_EQ(r[1], STR(" two"));
+  EXPECT_EQ(r[2], STR(" three"));
   r.clear();
 
-  size = Tokenize("one, two, three, ", ",", &r);
+  size = Tokenize(STR("one, two, three, "), STR(","), &r);
   EXPECT_EQ(4U, size);
   ASSERT_EQ(4U, r.size());
-  EXPECT_EQ(r[0], "one");
-  EXPECT_EQ(r[1], " two");
-  EXPECT_EQ(r[2], " three");
-  EXPECT_EQ(r[3], " ");
+  EXPECT_EQ(r[0], STR("one"));
+  EXPECT_EQ(r[1], STR(" two"));
+  EXPECT_EQ(r[2], STR(" three"));
+  EXPECT_EQ(r[3], STR(" "));
   r.clear();
 
-  size = Tokenize("one, two, three,", ",", &r);
+  size = Tokenize(STR("one, two, three,"), STR(","), &r);
   EXPECT_EQ(3U, size);
   ASSERT_EQ(3U, r.size());
-  EXPECT_EQ(r[0], "one");
-  EXPECT_EQ(r[1], " two");
-  EXPECT_EQ(r[2], " three");
+  EXPECT_EQ(r[0], STR("one"));
+  EXPECT_EQ(r[1], STR(" two"));
+  EXPECT_EQ(r[2], STR(" three"));
   r.clear();
 
-  size = Tokenize("", ",", &r);
+  size = Tokenize(STR(""), STR(","), &r);
   EXPECT_EQ(0U, size);
   ASSERT_EQ(0U, r.size());
   r.clear();
 
-  size = Tokenize(",", ",", &r);
+  size = Tokenize(STR(","), STR(","), &r);
   EXPECT_EQ(0U, size);
   ASSERT_EQ(0U, r.size());
   r.clear();
 
-  size = Tokenize(",;:.", ".:;,", &r);
+  size = Tokenize(STR(",;:."), STR(".:;,"), &r);
   EXPECT_EQ(0U, size);
   ASSERT_EQ(0U, r.size());
   r.clear();
 
-  size = Tokenize("\t\ta\t", "\t", &r);
+  size = Tokenize(STR("\t\ta\t"), STR("\t"), &r);
   EXPECT_EQ(1U, size);
   ASSERT_EQ(1U, r.size());
-  EXPECT_EQ(r[0], "a");
+  EXPECT_EQ(r[0], STR("a"));
   r.clear();
 
-  size = Tokenize("\ta\t\nb\tcc", "\n", &r);
+  size = Tokenize(STR("\ta\t\nb\tcc"), STR("\n"), &r);
   EXPECT_EQ(2U, size);
   ASSERT_EQ(2U, r.size());
-  EXPECT_EQ(r[0], "\ta\t");
-  EXPECT_EQ(r[1], "b\tcc");
+  EXPECT_EQ(r[0], STR("\ta\t"));
+  EXPECT_EQ(r[1], STR("b\tcc"));
   r.clear();
 }
 
+TEST(StringUtilTest, TokenizeStdString) {
+  TokenizeTest<std::string>();
+}
+
+TEST(StringUtilTest, TokenizeStringPiece) {
+  TokenizeTest<base::StringPiece>();
+}
+
 // Test for JoinString
 TEST(StringUtilTest, JoinString) {
   std::vector<std::string> in;