/ src / processor / tokenize.cc
tokenize.cc
 1  // Copyright 2010 Google LLC
 2  //
 3  // Redistribution and use in source and binary forms, with or without
 4  // modification, are permitted provided that the following conditions are
 5  // met:
 6  //
 7  //     * Redistributions of source code must retain the above copyright
 8  // notice, this list of conditions and the following disclaimer.
 9  //     * Redistributions in binary form must reproduce the above
10  // copyright notice, this list of conditions and the following disclaimer
11  // in the documentation and/or other materials provided with the
12  // distribution.
13  //     * Neither the name of Google LLC nor the names of its
14  // contributors may be used to endorse or promote products derived from
15  // this software without specific prior written permission.
16  //
17  // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18  // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19  // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20  // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21  // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22  // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23  // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24  // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25  // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26  // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27  // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28  
29  #ifdef HAVE_CONFIG_H
30  #include <config.h>  // Must come first
31  #endif
32  
33  #include <string.h>
34  
35  #include <string>
36  #include <vector>
37  
38  #include "common/using_std_string.h"
39  
40  namespace google_breakpad {
41  
42  #ifdef _MSC_VER
43  #define strtok_r strtok_s
44  #endif
45  
46  using std::vector;
47  
48  bool Tokenize(char* line,
49                const char* separators,
50                int max_tokens,
51                vector<char*>* tokens) {
52    tokens->clear();
53    tokens->reserve(max_tokens);
54  
55    int remaining = max_tokens;
56  
57    // Split tokens on the separator character.
58    // strip them out before exhausting max_tokens.
59    char* save_ptr;
60    char* token = strtok_r(line, separators, &save_ptr);
61    while (token && --remaining > 0) {
62      tokens->push_back(token);
63      if (remaining > 1)
64        token = strtok_r(NULL, separators, &save_ptr);
65    }
66  
67    // If there's anything left, just add it as a single token.
68    if (remaining == 0 && (token = strtok_r(NULL, "\r\n", &save_ptr))) {
69      tokens->push_back(token);
70    }
71  
72    return tokens->size() == static_cast<unsigned int>(max_tokens);
73  }
74  
75  void StringToVector(const string& str, vector<char>& vec) {
76    vec.resize(str.length() + 1);
77    std::copy(str.begin(), str.end(), vec.begin());
78    vec[str.length()] = '\0';
79  }
80  
81  } // namespace google_breakpad