/* ****************************************************************** * * Marcos Abreu Bento * * marcosbento@gmail.com * * ------------------------------------------------------------------ * * Experiments with string tokenizer and Boost * * ****************************************************************** */ #include #include #include #include #include #include #include /* * string tokenizer using standard input stream iterators * * NOTE: considers non-visible characters (e.g. " \n\t") as separators * ******************************************************************* */ void tokenize( std::vector& tokens, const std::string& source ) { std::stringstream strstr(source); std::istream_iterator it(strstr); std::istream_iterator end; tokens.assign(it, end); } /* * boost related type alias * ******************************************************************* */ typedef boost::char_separator char_separator_t; typedef boost::tokenizer< char_separator_t > char_tokenizer_t; /* * string tokenizer using boost tokenizing facilities * ******************************************************************* */ void tokenize( std::vector& tokens, const std::string& source, const std::string& delimiters ) { char_separator_t separator( delimiters.c_str() ); char_tokenizer_t tokenizer(source, separator); BOOST_FOREACH( std::string s, tokenizer ) tokens.push_back( s ); } /* * vector printing with output stream iterator * ******************************************************************* */ template void print_vector( const std::vector& v ) { std::ostream_iterator output( std::cout, "\n" ); std::copy( v.begin(), v.end(), output ); } /* * Main driver * * USAGE: Tokenizer < INPUT > OUT * ******************************************************************* */ int main() { std::string line; while( std::getline(std::cin, line) ) { std::vector tokens; tokenize( tokens, line, " .,:;{}()<>\\\t\n\""); print_vector( tokens ); } return 0; }