From 99b3e852e553103ff75ed38117baa5bce2befbdb Mon Sep 17 00:00:00 2001 From: David Wagner Date: Thu, 19 Mar 2015 16:35:47 +0100 Subject: Re-implement the Tokenizer class from scratch It didn't have a license header. Even though we received explicit authorization to use it, it wasn't a comfortable situation. This is an original implementation that only keeps the APIs we use in the parameter-framework and keep them identical (except for a small exception). The behaviour is also exactly the same. Change-Id: I85a69c76027ee026a693d79cd19edd3b86796f9d Signed-off-by: David Wagner --- utility/Tokenizer.cpp | 200 +++++++++++++++++++------------------------------- 1 file changed, 75 insertions(+), 125 deletions(-) (limited to 'utility/Tokenizer.cpp') diff --git a/utility/Tokenizer.cpp b/utility/Tokenizer.cpp index 9ea4ea4..a4cfcf0 100644 --- a/utility/Tokenizer.cpp +++ b/utility/Tokenizer.cpp @@ -1,125 +1,75 @@ -/////////////////////////////////////////////////////////////////////////////// -// Tokenizer.cpp -// ============= -// General purpose string tokenizer (C++ string version) -// -// The default delimiters are space(" "), tab(\t, \v), newline(\n), -// carriage return(\r), and form feed(\f). -// If you want to use different delimiters, then use setDelimiter() to override -// the delimiters. Note that the delimiter string can hold multiple characters. -// -// AUTHOR: Song Ho Ahn (song.ahn@gmail.com) -// CREATED: 2005-05-25 -// UPDATED: 2011-03-08 -/////////////////////////////////////////////////////////////////////////////// - -#include "Tokenizer.h" - - -/////////////////////////////////////////////////////////////////////////////// -// constructor -/////////////////////////////////////////////////////////////////////////////// -Tokenizer::Tokenizer() : buffer(""), token(""), delimiter(DEFAULT_DELIMITER) -{ - currPos = buffer.begin(); -} - -Tokenizer::Tokenizer(const std::string& str, const std::string& delimiter) : buffer(str), token(""), delimiter(delimiter) -{ - currPos = buffer.begin(); -} - - - -/////////////////////////////////////////////////////////////////////////////// -// destructor -/////////////////////////////////////////////////////////////////////////////// -Tokenizer::~Tokenizer() -{ -} - - - -/////////////////////////////////////////////////////////////////////////////// -// reset string buffer, delimiter and the currsor position -/////////////////////////////////////////////////////////////////////////////// -void Tokenizer::set(const std::string& str, const std::string& delimiter) -{ - this->buffer = str; - this->delimiter = delimiter; - this->currPos = buffer.begin(); -} - -void Tokenizer::setString(const std::string& str) -{ - this->buffer = str; - this->currPos = buffer.begin(); -} - -void Tokenizer::setDelimiter(const std::string& delimiter) -{ - this->delimiter = delimiter; - this->currPos = buffer.begin(); -} - - - -/////////////////////////////////////////////////////////////////////////////// -// return the next token -// If cannot find a token anymore, return "". -/////////////////////////////////////////////////////////////////////////////// -std::string Tokenizer::next() -{ - if(buffer.size() <= 0) return ""; // skip if buffer is empty - - token.clear(); // reset token string - - this->skipDelimiter(); // skip leading delimiters - - // append each char to token string until it meets delimiter - while(currPos != buffer.end() && !isDelimiter(*currPos)) - { - token += *currPos; - ++currPos; - } - return token; -} - - - -/////////////////////////////////////////////////////////////////////////////// -// skip ang leading delimiters -/////////////////////////////////////////////////////////////////////////////// -void Tokenizer::skipDelimiter() -{ - while(currPos != buffer.end() && isDelimiter(*currPos)) - ++currPos; -} - - - -/////////////////////////////////////////////////////////////////////////////// -// return true if the current character is delimiter -/////////////////////////////////////////////////////////////////////////////// -bool Tokenizer::isDelimiter(char c) -{ - return (delimiter.find(c) != std::string::npos); -} - - - -/////////////////////////////////////////////////////////////////////////////// -// split the input string into multiple tokens -// This function scans tokens from the current cursor position. -/////////////////////////////////////////////////////////////////////////////// -std::vector Tokenizer::split() -{ - std::vector tokens; - std::string token; - while((token = this->next()) != "") - { - tokens.push_back(token); - } - - return tokens; -} +/* + * Copyright (c) 2015, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "Tokenizer.h" + +using std::string; +using std::vector; + +const string Tokenizer::defaultDelimiters = " \n\r\t\v\f"; + +Tokenizer::Tokenizer(const string &input, const string &delimiters) + : _input(input), _delimiters(delimiters), _position(0) +{ +} + +string Tokenizer::next() +{ + string token; + + // Skip all leading delimiters + string::size_type tokenStart = _input.find_first_not_of(_delimiters, _position); + + // Special case if there isn't any token anymore (string::substr's + // throws when pos==npos) + if (tokenStart == string::npos) { + return ""; + } + + // Starting from the token's start, find the first delimiter + string::size_type tokenEnd = _input.find_first_of(_delimiters, tokenStart); + + _position = tokenEnd; + + return _input.substr(tokenStart, tokenEnd - tokenStart); +} + +vector Tokenizer::split() +{ + vector result; + string token; + + while (true) { + token = next(); + if (token.empty()) { + return result; + } + result.push_back(token); + } +} -- cgit v1.1