Split a string using C++11

Question

What would be easiest method to split a string using c++11?

I've seen the method used by this post, but I feel that there ought to be a less verbose way of doing it using the new standard.

Edit: I would like to have a vector<string> as a result and be able to delimitate on a single character.

Answer 1

std::regex_token_iterator performs generic tokenization based on a regex. It may or may not be overkill for doing simple splitting on a single character, but it works and is not too verbose:

std::vector<std::string> split(const string& input, const string& regex) {
    // passing -1 as the submatch index parameter performs splitting
    std::regex re(regex);
    std::sregex_token_iterator
        first{input.begin(), input.end(), re, -1},
        last;
    return {first, last};
}

Answer 2

Here is a (maybe less verbose) way to split string (based on the post you mentioned).

#include <string>
#include <sstream>
#include <vector>
std::vector<std::string> split(const std::string &s, char delim) {
  std::stringstream ss(s);
  std::string item;
  std::vector<std::string> elems;
  while (std::getline(ss, item, delim)) {
    elems.push_back(item);
    // elems.push_back(std::move(item)); // if C++11 (based on comment from @mchiasson)
  }
  return elems;
}

Answer 3

Here's an example of splitting a string and populating a vector with the extracted elements using boost.

#include <boost/algorithm/string.hpp>

std::string my_input("A,B,EE");
std::vector<std::string> results;

boost::algorithm::split(results, my_input, boost::is_any_of(","));

assert(results[0] == "A");
assert(results[1] == "B");
assert(results[2] == "EE");

Answer 4

Another regex solution inspired by other answers but hopefully shorter and easier to read:

std::string s{"String to split here, and here, and here,..."};
std::regex regex{R"([\s,]+)"}; // split on space and comma
std::sregex_token_iterator it{s.begin(), s.end(), regex, -1};
std::vector<std::string> words{it, {}};

Answer 5

I don't know if this is less verbose, but it might be easier to grok for those more seasoned in dynamic languages such as javascript. The only C++11 features it uses is auto and range-based for loop.

#include <string>
#include <cctype>
#include <iostream>
#include <vector>

using namespace std;

int main()
{
  string s = "hello  how    are you won't you tell me your name";
  vector<string> tokens;
  string token;

  for (const auto& c: s) {
    if (!isspace(c))
      token += c;
    else {
      if (token.length()) tokens.push_back(token);
      token.clear();
    }
  }

  if (token.length()) tokens.push_back(token);
     
  return 0;
}

Answer 6

#include <iostream>
#include <algorithm>
#include <vector>
#include <string>


using namespace std;

vector<string> split(const string& str, int delimiter(int) = ::isspace){
  vector<string> result;
  auto e=str.end();
  auto i=str.begin();
  while(i!=e){
    i=find_if_not(i,e, delimiter);
    if(i==e) break;
    auto j=find_if(i,e, delimiter);
    result.push_back(string(i,j));
    i=j;
  }
  return result;
}

int main(){
  string line;
  getline(cin,line);
  vector<string> result = split(line);
  for(auto s: result){
    cout<<s<<endl;
  }
}

Answer 7

My choice is boost::tokenizer but I didn't have any heavy tasks and test with huge data. Example from boost doc with lambda modification:

#include <iostream>
#include <boost/tokenizer.hpp>
#include <string>
#include <vector>

int main()
{
   using namespace std;
   using namespace boost;

   string s = "This is,  a test";
   vector<string> v;
   tokenizer<> tok(s);
   for_each (tok.begin(), tok.end(), [&v](const string & s) { v.push_back(s); } );
   // result 4 items: 1)This 2)is 3)a 4)test
   return 0;
}

Answer 8

#include <string>
#include <vector>
#include <sstream>

inline vector<string> split(const string& s) {
    vector<string> result;
    istringstream iss(s);
    for (string w; iss >> w; )
        result.push_back(w);
    return result;
}

Answer 9

This is my answer. Verbose, readable and efficient.

std::vector<std::string> tokenize(const std::string& s, char c) {
    auto end = s.cend();
    auto start = end;

    std::vector<std::string> v;
    for( auto it = s.cbegin(); it != end; ++it ) {
        if( *it != c ) {
            if( start == end )
                start = it;
            continue;
        }
        if( start != end ) {
            v.emplace_back(start, it);
            start = end;
        }
    }
    if( start != end )
        v.emplace_back(start, end);
    return v;
}

Answer 10

Here is a C++11 solution that uses only std::string::find(). The delimiter can be any number of characters long. Parsed tokens are output via an output iterator, which is typically a std::back_inserter in my code.

I have not tested this with UTF-8, but I expect it should work as long as the input and delimiter are both valid UTF-8 strings.

#include <string>

template<class Iter>
Iter splitStrings(const std::string &s, const std::string &delim, Iter out)
{
    if (delim.empty()) {
        *out++ = s;
        return out;
    }
    size_t a = 0, b = s.find(delim);
    for ( ; b != std::string::npos;
          a = b + delim.length(), b = s.find(delim, a))
    {
        *out++ = std::move(s.substr(a, b - a));
    }
    *out++ = std::move(s.substr(a, s.length() - a));
    return out;
}

Some test cases:

void test()
{
    std::vector<std::string> out;
    size_t counter;

    std::cout << "Empty input:" << std::endl;        
    out.clear();
    splitStrings("", ",", std::back_inserter(out));
    counter = 0;        
    for (auto i = out.begin(); i != out.end(); ++i, ++counter) {
        std::cout << counter << ": " << *i << std::endl;
    }

    std::cout << "Non-empty input, empty delimiter:" << std::endl;        
    out.clear();
    splitStrings("Hello, world!", "", std::back_inserter(out));
    counter = 0;        
    for (auto i = out.begin(); i != out.end(); ++i, ++counter) {
        std::cout << counter << ": " << *i << std::endl;
    }

    std::cout << "Non-empty input, non-empty delimiter"
                 ", no delimiter in string:" << std::endl;        
    out.clear();
    splitStrings("abxycdxyxydefxya", "xyz", std::back_inserter(out));
    counter = 0;        
    for (auto i = out.begin(); i != out.end(); ++i, ++counter) {
        std::cout << counter << ": " << *i << std::endl;
    }

    std::cout << "Non-empty input, non-empty delimiter"
                 ", delimiter exists string:" << std::endl;        
    out.clear();
    splitStrings("abxycdxy!!xydefxya", "xy", std::back_inserter(out));
    counter = 0;        
    for (auto i = out.begin(); i != out.end(); ++i, ++counter) {
        std::cout << counter << ": " << *i << std::endl;
    }

    std::cout << "Non-empty input, non-empty delimiter"
                 ", delimiter exists string"
                 ", input contains blank token:" << std::endl;        
    out.clear();
    splitStrings("abxycdxyxydefxya", "xy", std::back_inserter(out));
    counter = 0;        
    for (auto i = out.begin(); i != out.end(); ++i, ++counter) {
        std::cout << counter << ": " << *i << std::endl;
    }

    std::cout << "Non-empty input, non-empty delimiter"
                 ", delimiter exists string"
                 ", nothing after last delimiter:" << std::endl;        
    out.clear();
    splitStrings("abxycdxyxydefxy", "xy", std::back_inserter(out));
    counter = 0;        
    for (auto i = out.begin(); i != out.end(); ++i, ++counter) {
        std::cout << counter << ": " << *i << std::endl;
    }

    std::cout << "Non-empty input, non-empty delimiter"
                 ", only delimiter exists string:" << std::endl;        
    out.clear();
    splitStrings("xy", "xy", std::back_inserter(out));
    counter = 0;        
    for (auto i = out.begin(); i != out.end(); ++i, ++counter) {
        std::cout << counter << ": " << *i << std::endl;
    }
}

Expected output:

Empty input:
0: 
Non-empty input, empty delimiter:
0: Hello, world!
Non-empty input, non-empty delimiter, no delimiter in string:
0: abxycdxyxydefxya
Non-empty input, non-empty delimiter, delimiter exists string:
0: ab
1: cd
2: !!
3: def
4: a
Non-empty input, non-empty delimiter, delimiter exists string, input contains blank token:
0: ab
1: cd
2: 
3: def
4: a
Non-empty input, non-empty delimiter, delimiter exists string, nothing after last delimiter:
0: ab
1: cd
2: 
3: def
4: 
Non-empty input, non-empty delimiter, only delimiter exists string:
0: 
1: 

Answer 11

One possible way of doing this is finding all occurrences of the split string and storing locations to a list. Then count input string characters and when you get to a position where there is a 'search hit' in the position list then you jump forward by 'length of the split string'. This approach takes a split string of any length. Here is my tested and working solution.

#include <iostream>
#include <string>
#include <list>
#include <vector>

using namespace std;

vector<string> Split(string input_string, string search_string)
{
    list<int> search_hit_list;
    vector<string> word_list;
    size_t search_position, search_start = 0;

    // Find start positions of every substring occurence and store positions to a hit list.
    while ( (search_position = input_string.find(search_string, search_start) ) != string::npos) {
        search_hit_list.push_back(search_position);
        search_start = search_position + search_string.size();
    }

    // Iterate through hit list and reconstruct substring start and length positions
    int character_counter = 0;
    int start, length;

    for (auto hit_position : search_hit_list) {

        // Skip over substrings we are splitting with. This also skips over repeating substrings.
        if (character_counter == hit_position) {
            character_counter = character_counter + search_string.size();
            continue;
        }

        start = character_counter;
        character_counter = hit_position;
        length = character_counter - start;
        word_list.push_back(input_string.substr(start, length));
        character_counter = character_counter + search_string.size();
    }

    // If the search string is not found in the input string, then return the whole input_string.
    if (word_list.size() == 0) {
            word_list.push_back(input_string);
            return word_list;
    }
    // The last substring might be still be unprocessed, get it.
    if (character_counter < input_string.size()) {
        word_list.push_back(input_string.substr(character_counter, input_string.size() - character_counter));
    }

    return word_list;
}

int main() {

    vector<string> word_list;
    string search_string = " ";
    // search_string = "the";
    string text = "thetheThis is  some   text     to test  with the    split-thethe   function.";

    word_list = Split(text, search_string);

    for (auto item : word_list) {
        cout << "'" << item << "'" << endl;
    }

    cout << endl;
}