From f4ebe772e28369908c70e2e79f052d043c6f8eeb Mon Sep 17 00:00:00 2001 From: "brettw@chromium.org" Date: Sat, 2 Feb 2013 00:21:39 +0000 Subject: Move string tokenizer to base/strings. BUG= Review URL: https://codereview.chromium.org/12087091 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@180211 0039d316-1c4b-4281-b951-d872f2087c98 --- base/strings/string_tokenizer_unittest.cc | 234 ++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 base/strings/string_tokenizer_unittest.cc (limited to 'base/strings/string_tokenizer_unittest.cc') diff --git a/base/strings/string_tokenizer_unittest.cc b/base/strings/string_tokenizer_unittest.cc new file mode 100644 index 0000000..d391845 --- /dev/null +++ b/base/strings/string_tokenizer_unittest.cc @@ -0,0 +1,234 @@ +// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/strings/string_tokenizer.h" + +#include "testing/gtest/include/gtest/gtest.h" + +using std::string; + +namespace base { + +namespace { + +TEST(StringTokenizerTest, Simple) { + string input = "this is a test"; + StringTokenizer t(input, " "); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("this"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("is"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("a"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("test"), t.token()); + + EXPECT_FALSE(t.GetNext()); +} + +TEST(StringTokenizerTest, Reset) { + string input = "this is a test"; + StringTokenizer t(input, " "); + + for (int i = 0; i < 2; ++i) { + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("this"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("is"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("a"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("test"), t.token()); + + EXPECT_FALSE(t.GetNext()); + t.Reset(); + } +} + +TEST(StringTokenizerTest, RetDelims) { + string input = "this is a test"; + StringTokenizer t(input, " "); + t.set_options(StringTokenizer::RETURN_DELIMS); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("this"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string(" "), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("is"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string(" "), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("a"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string(" "), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("test"), t.token()); + + EXPECT_FALSE(t.GetNext()); +} + +TEST(StringTokenizerTest, ManyDelims) { + string input = "this: is, a-test"; + StringTokenizer t(input, ": ,-"); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("this"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("is"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("a"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("test"), t.token()); + + EXPECT_FALSE(t.GetNext()); +} + +TEST(StringTokenizerTest, ParseHeader) { + string input = "Content-Type: text/html ; charset=UTF-8"; + StringTokenizer t(input, ": ;="); + t.set_options(StringTokenizer::RETURN_DELIMS); + + EXPECT_TRUE(t.GetNext()); + EXPECT_FALSE(t.token_is_delim()); + EXPECT_EQ(string("Content-Type"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_TRUE(t.token_is_delim()); + EXPECT_EQ(string(":"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_TRUE(t.token_is_delim()); + EXPECT_EQ(string(" "), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_FALSE(t.token_is_delim()); + EXPECT_EQ(string("text/html"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_TRUE(t.token_is_delim()); + EXPECT_EQ(string(" "), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_TRUE(t.token_is_delim()); + EXPECT_EQ(string(";"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_TRUE(t.token_is_delim()); + EXPECT_EQ(string(" "), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_FALSE(t.token_is_delim()); + EXPECT_EQ(string("charset"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_TRUE(t.token_is_delim()); + EXPECT_EQ(string("="), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_FALSE(t.token_is_delim()); + EXPECT_EQ(string("UTF-8"), t.token()); + + EXPECT_FALSE(t.GetNext()); + EXPECT_FALSE(t.token_is_delim()); +} + +TEST(StringTokenizerTest, ParseQuotedString) { + string input = "foo bar 'hello world' baz"; + StringTokenizer t(input, " "); + t.set_quote_chars("'"); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("foo"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("bar"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("'hello world'"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("baz"), t.token()); + + EXPECT_FALSE(t.GetNext()); +} + +TEST(StringTokenizerTest, ParseQuotedString_Malformed) { + string input = "bar 'hello wo"; + StringTokenizer t(input, " "); + t.set_quote_chars("'"); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("bar"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("'hello wo"), t.token()); + + EXPECT_FALSE(t.GetNext()); +} + +TEST(StringTokenizerTest, ParseQuotedString_Multiple) { + string input = "bar 'hel\"lo\" wo' baz\""; + StringTokenizer t(input, " "); + t.set_quote_chars("'\""); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("bar"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("'hel\"lo\" wo'"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("baz\""), t.token()); + + EXPECT_FALSE(t.GetNext()); +} + +TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes) { + string input = "foo 'don\\'t do that'"; + StringTokenizer t(input, " "); + t.set_quote_chars("'"); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("foo"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("'don\\'t do that'"), t.token()); + + EXPECT_FALSE(t.GetNext()); +} + +TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes2) { + string input = "foo='a, b', bar"; + StringTokenizer t(input, ", "); + t.set_quote_chars("'"); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("foo='a, b'"), t.token()); + + EXPECT_TRUE(t.GetNext()); + EXPECT_EQ(string("bar"), t.token()); + + EXPECT_FALSE(t.GetNext()); +} + +} // namespace + +} // namespace base -- cgit v1.1