Cppcheck
tokenlist.h
Go to the documentation of this file.
1 /*
2  * Cppcheck - A tool for static C/C++ code analysis
3  * Copyright (C) 2007-2024 Cppcheck team.
4  *
5  * This program is free software: you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation, either version 3 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  */
18 
19 //---------------------------------------------------------------------------
20 #ifndef tokenlistH
21 #define tokenlistH
22 //---------------------------------------------------------------------------
23 
24 #include "config.h"
25 #include "standards.h"
26 
27 #include <cstddef>
28 #include <iosfwd>
29 #include <string>
30 #include <vector>
31 
32 class Token;
33 class TokenList;
34 class Settings;
35 
36 namespace simplecpp {
37  class TokenList;
38 }
39 
40 /// @addtogroup Core
41 /// @{
42 
43 /**
44  * @brief This struct stores pointers to the front and back tokens of the list this token is in.
45  */
47  explicit TokensFrontBack(const TokenList& list) : list(list) {}
49  Token* back{};
50  const TokenList& list;
51 };
52 
54 public:
55  // TODO: pass settings as reference
56  explicit TokenList(const Settings* settings);
57  ~TokenList();
58 
59  TokenList(const TokenList &) = delete;
60  TokenList &operator=(const TokenList &) = delete;
61 
62  /** @return the source file path. e.g. "file.cpp" */
63  const std::string& getSourceFilePath() const;
64 
65  /** @return true if the code is C */
66  bool isC() const;
67 
68  /** @return true if the code is C++ */
69  bool isCPP() const;
70 
72 
73  /**
74  * Delete all tokens in given token list
75  * @param tok token list to delete
76  */
77  static void deleteTokens(Token *tok);
78 
79  void addtoken(const std::string& str, const nonneg int lineno, const nonneg int column, const nonneg int fileno, bool split = false);
80  void addtoken(const std::string& str, const Token *locationTok);
81 
82  void addtoken(const Token *tok, const nonneg int lineno, const nonneg int column, const nonneg int fileno);
83  void addtoken(const Token *tok, const Token *locationTok);
84  void addtoken(const Token *tok);
85 
86  static void insertTokens(Token *dest, const Token *src, nonneg int n);
87 
88  /**
89  * Copy tokens.
90  * @param dest destination token where copied tokens will be inserted after
91  * @param first first token to copy
92  * @param last last token to copy
93  * @param one_line true=>copy all tokens to the same line as dest. false=>copy all tokens to dest while keeping the 'line breaks'
94  * @return new location of last token copied
95  */
96  static Token *copyTokens(Token *dest, const Token *first, const Token *last, bool one_line = true);
97 
98  /**
99  * Create tokens from code.
100  * The code must be preprocessed first:
101  * - multiline strings are not handled.
102  * - UTF in the code are not handled.
103  * - comments are not handled.
104  * @param code input stream for code
105  * @param file0 source file name
106  */
107  bool createTokens(std::istream &code, const std::string& file0);
108  bool createTokens(std::istream &code, Standards::Language lang);
109 
110  void createTokens(simplecpp::TokenList&& tokenList);
111 
112  /** Deallocate list */
113  void deallocateTokens();
114 
115  /** append file name if seen the first time; return its index in any case */
116  int appendFileIfNew(std::string fileName);
117 
118  /** get first token of list */
119  const Token *front() const {
120  return mTokensFrontBack.front;
121  }
122  // NOLINTNEXTLINE(readability-make-member-function-const) - do not allow usage of mutable pointer from const object
124  return mTokensFrontBack.front;
125  }
126 
127  /** get last token of list */
128  const Token *back() const {
129  return mTokensFrontBack.back;
130  }
131  // NOLINTNEXTLINE(readability-make-member-function-const) - do not allow usage of mutable pointer from const object
132  Token *back() {
133  return mTokensFrontBack.back;
134  }
135 
136  /**
137  * Get filenames (the sourcefile + the files it include).
138  * The first filename is the filename for the sourcefile
139  * @return vector with filenames
140  */
141  const std::vector<std::string>& getFiles() const {
142  return mFiles;
143  }
144 
145  std::string getOrigFile(const Token *tok) const;
146 
147  /**
148  * get filename for given token
149  * @param tok The given token
150  * @return filename for the given token
151  */
152  const std::string& file(const Token *tok) const;
153 
154  /**
155  * Get file:line for a given token
156  * @param tok given token
157  * @return location for given token
158  */
159  std::string fileLine(const Token *tok) const;
160 
161  /**
162  * Calculates a hash of the token list used to compare multiple
163  * token lists with each other as quickly as possible.
164  */
165  std::size_t calculateHash() const;
166 
167  /**
168  * Create abstract syntax tree.
169  */
170  void createAst() const;
171 
172  /**
173  * Check abstract syntax tree.
174  * Throws InternalError on failure
175  */
176  void validateAst(bool print) const;
177 
178  /**
179  * Verify that the given token is an element of the tokenlist.
180  * That method is implemented for debugging purposes.
181  * @param[in] tok token to be checked
182  * \return true if token was found in tokenlist, false else. In case of nullptr true is returned.
183  */
184  bool validateToken(const Token* tok) const;
185 
186  /**
187  * Convert platform dependent types to standard types.
188  * 32 bits: size_t -> unsigned long
189  * 64 bits: size_t -> unsigned long long
190  */
192 
193  /**
194  * Collapse compound standard types into a single token.
195  * unsigned long long int => long _isUnsigned=true,_isLong=true
196  */
198 
199  void clangSetOrigFiles();
200 
201  bool isKeyword(const std::string &str) const;
202 
203 private:
204  void determineCppC();
205 
206  bool createTokensInternal(std::istream &code, const std::string& file0);
207 
208  /** Token list */
210 
211  /** filenames for the tokenized source code (source + included) */
212  std::vector<std::string> mFiles;
213 
214  /** Original filenames for the tokenized source code (source + included) */
215  std::vector<std::string> mOrigFiles;
216 
217  /** settings */
218  const Settings* const mSettings{};
219 
220  /** File is known to be C/C++ code */
221  Standards::Language mLang{Standards::Language::None};
222 };
223 
224 /// @}
225 
226 const Token* isLambdaCaptureList(const Token* tok);
228 
229 //---------------------------------------------------------------------------
230 #endif // tokenlistH
This is just a container for general settings so that we don't need to pass individual values to func...
Definition: settings.h:95
bool isC() const
TokensFrontBack mTokensFrontBack
Token list.
Definition: tokenlist.h:209
std::string fileLine(const Token *tok) const
Get file:line for a given token.
void createAst() const
Create abstract syntax tree.
TokenList & operator=(const TokenList &)=delete
const Token * back() const
get last token of list
Definition: tokenlist.h:128
void simplifyPlatformTypes()
Convert platform dependent types to standard types.
bool validateToken(const Token *tok) const
Verify that the given token is an element of the tokenlist.
TokenList(const TokenList &)=delete
Token * front()
Definition: tokenlist.h:123
bool isKeyword(const std::string &str) const
Token * back()
Definition: tokenlist.h:132
void setLang(Standards::Language lang)
const std::string & file(const Token *tok) const
get filename for given token
bool isCPP() const
void simplifyStdType()
Collapse compound standard types into a single token.
const std::vector< std::string > & getFiles() const
Get filenames (the sourcefile + the files it include).
Definition: tokenlist.h:141
const Token * front() const
get first token of list
Definition: tokenlist.h:119
std::vector< std::string > mOrigFiles
Original filenames for the tokenized source code (source + included)
Definition: tokenlist.h:215
std::vector< std::string > mFiles
filenames for the tokenized source code (source + included)
Definition: tokenlist.h:212
std::string getOrigFile(const Token *tok) const
void validateAst(bool print) const
Check abstract syntax tree.
The token list that the TokenList generates is a linked-list of this class.
Definition: token.h:150
#define CPPCHECKLIB
Definition: config.h:35
#define nonneg
Definition: config.h:138
static std::vector< std::string > split(const std::string &str, const std::string &sep=" ")
Definition: cppcheck.cpp:113
This struct stores pointers to the front and back tokens of the list this token is in.
Definition: tokenlist.h:46
Token * back
Definition: tokenlist.h:49
const TokenList & list
Definition: tokenlist.h:50
TokensFrontBack(const TokenList &list)
Definition: tokenlist.h:47
Token * front
Definition: tokenlist.h:48
const Token * findLambdaEndTokenWithoutAST(const Token *tok)
const Token * isLambdaCaptureList(const Token *tok)