summaryrefslogtreecommitdiffstats
path: root/tools/gn/tokenizer_unittest.cc
diff options
context:
space:
mode:
Diffstat (limited to 'tools/gn/tokenizer_unittest.cc')
-rw-r--r--tools/gn/tokenizer_unittest.cc162
1 files changed, 0 insertions, 162 deletions
diff --git a/tools/gn/tokenizer_unittest.cc b/tools/gn/tokenizer_unittest.cc
deleted file mode 100644
index d1a6788..0000000
--- a/tools/gn/tokenizer_unittest.cc
+++ /dev/null
@@ -1,162 +0,0 @@
-// Copyright (c) 2013 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "testing/gtest/include/gtest/gtest.h"
-#include "tools/gn/input_file.h"
-#include "tools/gn/token.h"
-#include "tools/gn/tokenizer.h"
-
-namespace {
-
-struct TokenExpectation {
- Token::Type type;
- const char* value;
-};
-
-template<size_t len>
-bool CheckTokenizer(const char* input, const TokenExpectation (&expect)[len]) {
- InputFile input_file(SourceFile("/test"));
- input_file.SetContents(input);
-
- Err err;
- std::vector<Token> results = Tokenizer::Tokenize(&input_file, &err);
-
- if (results.size() != len)
- return false;
- for (size_t i = 0; i < len; i++) {
- if (expect[i].type != results[i].type())
- return false;
- if (expect[i].value != results[i].value())
- return false;
- }
- return true;
-}
-
-} // namespace
-
-TEST(Tokenizer, Empty) {
- InputFile empty_string_input(SourceFile("/test"));
- empty_string_input.SetContents("");
-
- Err err;
- std::vector<Token> results = Tokenizer::Tokenize(&empty_string_input, &err);
- EXPECT_TRUE(results.empty());
-
- InputFile whitespace_input(SourceFile("/test"));
- whitespace_input.SetContents(" \n\r");
-
- results = Tokenizer::Tokenize(&whitespace_input, &err);
- EXPECT_TRUE(results.empty());
-}
-
-TEST(Tokenizer, Identifier) {
- TokenExpectation one_ident[] = {
- { Token::IDENTIFIER, "foo" }
- };
- EXPECT_TRUE(CheckTokenizer(" foo ", one_ident));
-}
-
-TEST(Tokenizer, Integer) {
- TokenExpectation integers[] = {
- { Token::INTEGER, "123" },
- { Token::INTEGER, "-123" }
- };
- EXPECT_TRUE(CheckTokenizer(" 123 -123 ", integers));
-}
-
-TEST(Tokenizer, String) {
- TokenExpectation strings[] = {
- { Token::STRING, "\"foo\"" },
- { Token::STRING, "\"bar\\\"baz\"" },
- { Token::STRING, "\"asdf\\\\\"" }
- };
- EXPECT_TRUE(CheckTokenizer(" \"foo\" \"bar\\\"baz\" \"asdf\\\\\" ",
- strings));
-}
-
-TEST(Tokenizer, Operator) {
- TokenExpectation operators[] = {
- { Token::OPERATOR, "-" },
- { Token::OPERATOR, "+" },
- { Token::OPERATOR, "=" },
- { Token::OPERATOR, "+=" },
- { Token::OPERATOR, "-=" },
- { Token::OPERATOR, "!=" },
- { Token::OPERATOR, "==" },
- { Token::OPERATOR, "<" },
- { Token::OPERATOR, ">" },
- { Token::OPERATOR, "<=" },
- { Token::OPERATOR, ">=" },
- };
- EXPECT_TRUE(CheckTokenizer("- + = += -= != == < > <= >=",
- operators));
-}
-
-TEST(Tokenizer, Scoper) {
- TokenExpectation scopers[] = {
- { Token::SCOPER, "{" },
- { Token::SCOPER, "[" },
- { Token::SCOPER, "]" },
- { Token::SCOPER, "}" },
- { Token::SCOPER, "(" },
- { Token::SCOPER, ")" },
- };
- EXPECT_TRUE(CheckTokenizer("{[ ]} ()", scopers));
-}
-
-TEST(Tokenizer, FunctionCall) {
- TokenExpectation fn[] = {
- { Token::IDENTIFIER, "fun" },
- { Token::SCOPER, "(" },
- { Token::STRING, "\"foo\"" },
- { Token::SCOPER, ")" },
- { Token::SCOPER, "{" },
- { Token::IDENTIFIER, "foo" },
- { Token::OPERATOR, "=" },
- { Token::INTEGER, "12" },
- { Token::SCOPER, "}" },
- };
- EXPECT_TRUE(CheckTokenizer("fun(\"foo\") {\nfoo = 12}", fn));
-}
-
-TEST(Tokenizer, StringUnescaping) {
- InputFile input(SourceFile("/test"));
- input.SetContents("\"asd\\\"f\" \"\"");
- Err err;
- std::vector<Token> results = Tokenizer::Tokenize(&input, &err);
-
- ASSERT_EQ(2u, results.size());
- EXPECT_EQ("asd\"f", results[0].StringValue());
- EXPECT_EQ("", results[1].StringValue());
-}
-
-TEST(Tokenizer, Locations) {
- InputFile input(SourceFile("/test"));
- input.SetContents("1 2 \"three\"\n 4");
- Err err;
- std::vector<Token> results = Tokenizer::Tokenize(&input, &err);
-
- ASSERT_EQ(4u, results.size());
- ASSERT_TRUE(results[0].location() == Location(&input, 1, 1));
- ASSERT_TRUE(results[1].location() == Location(&input, 1, 3));
- ASSERT_TRUE(results[2].location() == Location(&input, 1, 5));
- ASSERT_TRUE(results[3].location() == Location(&input, 2, 3));
-}
-
-TEST(Tokenizer, ByteOffsetOfNthLine) {
- EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine("foo", 1));
-
- // Windows and Posix have different line endings, so check the byte at the
- // location rather than the offset.
- char input1[] = "aaa\nxaa\n\nya";
- EXPECT_EQ('x', input1[Tokenizer::ByteOffsetOfNthLine(input1, 2)]);
- EXPECT_EQ('y', input1[Tokenizer::ByteOffsetOfNthLine(input1, 4)]);
-
- char input2[3];
- input2[0] = 'a';
- input2[1] = '\n'; // Manually set to avoid Windows double-byte endings.
- input2[2] = 0;
- EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine(input2, 1));
- EXPECT_EQ(2u, Tokenizer::ByteOffsetOfNthLine(input2, 2));
-}