diff options
Diffstat (limited to 'tools/gn/tokenizer_unittest.cc')
-rw-r--r-- | tools/gn/tokenizer_unittest.cc | 162 |
1 files changed, 162 insertions, 0 deletions
diff --git a/tools/gn/tokenizer_unittest.cc b/tools/gn/tokenizer_unittest.cc new file mode 100644 index 0000000..d1a6788 --- /dev/null +++ b/tools/gn/tokenizer_unittest.cc @@ -0,0 +1,162 @@ +// Copyright (c) 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "testing/gtest/include/gtest/gtest.h" +#include "tools/gn/input_file.h" +#include "tools/gn/token.h" +#include "tools/gn/tokenizer.h" + +namespace { + +struct TokenExpectation { + Token::Type type; + const char* value; +}; + +template<size_t len> +bool CheckTokenizer(const char* input, const TokenExpectation (&expect)[len]) { + InputFile input_file(SourceFile("/test")); + input_file.SetContents(input); + + Err err; + std::vector<Token> results = Tokenizer::Tokenize(&input_file, &err); + + if (results.size() != len) + return false; + for (size_t i = 0; i < len; i++) { + if (expect[i].type != results[i].type()) + return false; + if (expect[i].value != results[i].value()) + return false; + } + return true; +} + +} // namespace + +TEST(Tokenizer, Empty) { + InputFile empty_string_input(SourceFile("/test")); + empty_string_input.SetContents(""); + + Err err; + std::vector<Token> results = Tokenizer::Tokenize(&empty_string_input, &err); + EXPECT_TRUE(results.empty()); + + InputFile whitespace_input(SourceFile("/test")); + whitespace_input.SetContents(" \n\r"); + + results = Tokenizer::Tokenize(&whitespace_input, &err); + EXPECT_TRUE(results.empty()); +} + +TEST(Tokenizer, Identifier) { + TokenExpectation one_ident[] = { + { Token::IDENTIFIER, "foo" } + }; + EXPECT_TRUE(CheckTokenizer(" foo ", one_ident)); +} + +TEST(Tokenizer, Integer) { + TokenExpectation integers[] = { + { Token::INTEGER, "123" }, + { Token::INTEGER, "-123" } + }; + EXPECT_TRUE(CheckTokenizer(" 123 -123 ", integers)); +} + +TEST(Tokenizer, String) { + TokenExpectation strings[] = { + { Token::STRING, "\"foo\"" }, + { Token::STRING, "\"bar\\\"baz\"" }, + { Token::STRING, "\"asdf\\\\\"" } + }; + EXPECT_TRUE(CheckTokenizer(" \"foo\" \"bar\\\"baz\" \"asdf\\\\\" ", + strings)); +} + +TEST(Tokenizer, Operator) { + TokenExpectation operators[] = { + { Token::OPERATOR, "-" }, + { Token::OPERATOR, "+" }, + { Token::OPERATOR, "=" }, + { Token::OPERATOR, "+=" }, + { Token::OPERATOR, "-=" }, + { Token::OPERATOR, "!=" }, + { Token::OPERATOR, "==" }, + { Token::OPERATOR, "<" }, + { Token::OPERATOR, ">" }, + { Token::OPERATOR, "<=" }, + { Token::OPERATOR, ">=" }, + }; + EXPECT_TRUE(CheckTokenizer("- + = += -= != == < > <= >=", + operators)); +} + +TEST(Tokenizer, Scoper) { + TokenExpectation scopers[] = { + { Token::SCOPER, "{" }, + { Token::SCOPER, "[" }, + { Token::SCOPER, "]" }, + { Token::SCOPER, "}" }, + { Token::SCOPER, "(" }, + { Token::SCOPER, ")" }, + }; + EXPECT_TRUE(CheckTokenizer("{[ ]} ()", scopers)); +} + +TEST(Tokenizer, FunctionCall) { + TokenExpectation fn[] = { + { Token::IDENTIFIER, "fun" }, + { Token::SCOPER, "(" }, + { Token::STRING, "\"foo\"" }, + { Token::SCOPER, ")" }, + { Token::SCOPER, "{" }, + { Token::IDENTIFIER, "foo" }, + { Token::OPERATOR, "=" }, + { Token::INTEGER, "12" }, + { Token::SCOPER, "}" }, + }; + EXPECT_TRUE(CheckTokenizer("fun(\"foo\") {\nfoo = 12}", fn)); +} + +TEST(Tokenizer, StringUnescaping) { + InputFile input(SourceFile("/test")); + input.SetContents("\"asd\\\"f\" \"\""); + Err err; + std::vector<Token> results = Tokenizer::Tokenize(&input, &err); + + ASSERT_EQ(2u, results.size()); + EXPECT_EQ("asd\"f", results[0].StringValue()); + EXPECT_EQ("", results[1].StringValue()); +} + +TEST(Tokenizer, Locations) { + InputFile input(SourceFile("/test")); + input.SetContents("1 2 \"three\"\n 4"); + Err err; + std::vector<Token> results = Tokenizer::Tokenize(&input, &err); + + ASSERT_EQ(4u, results.size()); + ASSERT_TRUE(results[0].location() == Location(&input, 1, 1)); + ASSERT_TRUE(results[1].location() == Location(&input, 1, 3)); + ASSERT_TRUE(results[2].location() == Location(&input, 1, 5)); + ASSERT_TRUE(results[3].location() == Location(&input, 2, 3)); +} + +TEST(Tokenizer, ByteOffsetOfNthLine) { + EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine("foo", 1)); + + // Windows and Posix have different line endings, so check the byte at the + // location rather than the offset. + char input1[] = "aaa\nxaa\n\nya"; + EXPECT_EQ('x', input1[Tokenizer::ByteOffsetOfNthLine(input1, 2)]); + EXPECT_EQ('y', input1[Tokenizer::ByteOffsetOfNthLine(input1, 4)]); + + char input2[3]; + input2[0] = 'a'; + input2[1] = '\n'; // Manually set to avoid Windows double-byte endings. + input2[2] = 0; + EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine(input2, 1)); + EXPECT_EQ(2u, Tokenizer::ByteOffsetOfNthLine(input2, 2)); +} |