tokenize.h

Go to the documentation of this file.

/*
 * Cppcheck - A tool for static C/C++ code analysis
 * Copyright (C) 2007-2024 Cppcheck team.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
//---------------------------------------------------------------------------
#ifndef tokenizeH
#define tokenizeH
//---------------------------------------------------------------------------
 
#include "config.h"
#include "tokenlist.h"
 
#include <iosfwd>
#include <list>
#include <map>
#include <string>
#include <vector>
 
class Settings;
class SymbolDatabase;
class TimerResults;
class Token;
class TemplateSimplifier;
class ErrorLogger;
struct Directive; // IWYU pragma: keep
enum class Severity;
 
/// @addtogroup Core
/// @{
 
/** @brief The main purpose is to tokenize the source code. It also has functions that simplify the token list */
class CPPCHECKLIB Tokenizer {
 
    friend class SymbolDatabase;
    friend class TemplateSimplifier;
 
    friend class TestSimplifyTemplate;
    friend class TestSimplifyTypedef;
    friend class TestTokenizer;
 
public:
    explicit Tokenizer(const Settings & settings, ErrorLogger &errorLogger);
    ~Tokenizer();
 
    void setTimerResults(TimerResults *tr) {
        mTimerResults = tr;
    }
 
    /** Is the code C. Used for bailouts */
    bool isC() const {
        return list.isC();
    }
 
    /** Is the code CPP. Used for bailouts */
    bool isCPP() const {
        return list.isCPP();
    }
 
    /**
     * Check if inner scope ends with a call to a noreturn function
     * \param endScopeToken The '}' token
     * \param unknown set to true if it's unknown if the scope is noreturn
     * \return true if scope ends with a function call that might be 'noreturn'
     */
    bool isScopeNoReturn(const Token *endScopeToken, bool *unknown = nullptr) const;
 
    bool simplifyTokens1(const std::string &configuration);
 
private:
    /** Set variable id */
    void setVarId();
    void setVarIdPass1();
    void setVarIdPass2();
 
    /**
     * Basic simplification of tokenlist
     *
     * @param FileName The filename to run; used to do
     * markup checks.
     *
     * @return false if there is an error that requires aborting
     * the checking of this file.
     */
    bool simplifyTokenList1(const char FileName[]);
 
    /**
     * If --check-headers=no has been given; then remove unneeded code in headers.
     * - All executable code.
     * - Unused types/variables/etc
     */
    void simplifyHeadersAndUnusedTemplates();
 
    /**
     * Remove extra "template" keywords that are not used by Cppcheck
     */
    void removeExtraTemplateKeywords();
 
 
    /** Split up template right angle brackets.
     * foo < bar < >> => foo < bar < > >
     */
    void splitTemplateRightAngleBrackets(bool check);
 
public:
    /**
     * Calculates sizeof value for given type.
     * @param type Token which will contain e.g. "int", "*", or string.
     * @return sizeof for given type, or 0 if it can't be calculated.
     */
    nonneg int sizeOfType(const Token* type) const;
    nonneg int sizeOfType(const std::string& type) const;
 
private:
    void simplifyDebug();
 
    /** Simplify assignment where rhs is a block : "x=({123;});" => "{x=123;}" */
    void simplifyAssignmentBlock();
 
    /** Insert array size where it isn't given */
    void arraySize();
    void arraySizeAfterValueFlow(); // cppcheck-suppress functionConst
 
    /** Simplify labels and 'case|default' syntaxes.
     */
    void simplifyLabelsCaseDefault();
 
    /** simplify case ranges (gcc extension)
     */
    void simplifyCaseRange();
 
    /** Remove macros in global scope */
    void removeMacrosInGlobalScope();
 
    void addSemicolonAfterUnknownMacro();
 
    // Remove C99 and CPP11 _Pragma(str)
    void removePragma();
 
    /** Remove undefined macro in class definition:
     * class DLLEXPORT Fred { };
     * class Fred FINAL : Base { };
     */
    void removeMacroInClassDef();
 
    /** Add parentheses for sizeof: sizeof x => sizeof(x) */
    void sizeofAddParentheses();
 
    /**
     * Simplify variable declarations (split up)
     * \param only_k_r_fpar Only simplify K&R function parameters
     */
    void simplifyVarDecl(const bool only_k_r_fpar);
    void simplifyVarDecl(Token * tokBegin, const Token * const tokEnd, const bool only_k_r_fpar); // cppcheck-suppress functionConst // has side effects
 
    /**
     * Simplify variable initialization
     * '; int *p(0);' => '; int *p = 0;'
     */
    void simplifyInitVar();
    static Token* initVar(Token* tok);
 
    /**
     * Simplify the location of "static" and "const" qualifiers in
     * a variable declaration or definition.
     * Example: "int static const a;" => "static const a;"
     * Example: "long long const static b;" => "static const long long b;"
     */
    void simplifyStaticConst();
 
    /**
     * Simplify multiple assignments.
     * Example: "a = b = c = 0;" => "a = 0; b = 0; c = 0;"
     */
    void simplifyVariableMultipleAssign();
 
    /**
     * Simplify the 'C Alternative Tokens'
     * Examples:
     * "if(s and t)" => "if(s && t)"
     * "while((r bitand s) and not t)" => while((r & s) && !t)"
     * "a and_eq b;" => "a &= b;"
     */
    bool simplifyCAlternativeTokens();
 
    /** Add braces to an if-block, for-block, etc.
     * @return true if no syntax errors
     */
    bool simplifyAddBraces();
 
    /** Add braces to an if-block, for-block, etc.
     * for command starting at token including else-block
     * @return last token of command
     *         or input token in case of an error where no braces are added
     *         or NULL when syntaxError is called
     */
    Token * simplifyAddBracesToCommand(Token * tok);
 
    /** Add pair of braces to an single if-block, else-block, for-block, etc.
     * for command starting at token
     * @return last token of command
     *         or input token in case of an error where no braces are added
     *         or NULL when syntaxError is called
     */
    Token * simplifyAddBracesPair(Token *tok, bool commandWithCondition);
 
    // Convert "using ...;" to corresponding typedef
    void simplifyUsingToTypedef();
 
    /**
     * typedef A mytype;
     * mytype c;
     *
     * Becomes:
     * typedef A mytype;
     * A c;
     */
    void simplifyTypedef();
    void simplifyTypedefCpp();
    /**
     * Move typedef token to the left og the expression
     */
    void simplifyTypedefLHS();
 
    /**
     */
    static bool isMemberFunction(const Token *openParen);
 
    /**
     */
    bool simplifyUsing();
    void simplifyUsingError(const Token* usingStart, const Token* usingEnd);
 
    /** Simplify useless C++ empty namespaces, like: 'namespace %name% { }'*/
    void simplifyEmptyNamespaces();
 
    /** Simplify "if else" */
    void elseif();
 
    /** Simplify C++17/C++20 if/switch/for initialization expression */
    void simplifyIfSwitchForInit();
 
    /**
     * Reduces "; ;" to ";", except in "( ; ; )"
     */
    void removeRedundantSemicolons();
 
    /** Struct simplification
     * "struct S { } s;" => "struct S { }; S s;"
     */
 
    void simplifyStructDecl();
 
    /**
     * Remove redundant parentheses:
     * - "((x))" => "(x)"
     * - "(function())" => "function()"
     * - "(delete x)" => "delete x"
     * - "(delete [] x)" => "delete [] x"
     * @return true if modifications to token-list are done.
     *         false if no modifications are done.
     */
    bool simplifyRedundantParentheses();
 
    /**
     * Simplify functions like "void f(x) int x; {"
     * into "void f(int x) {"
     */
    void simplifyFunctionParameters();
 
    /** Simplify function level try blocks:
     *  Convert "void f() try {} catch (int) {}"
     *  to "void f() { try {} catch (int) {} }"
     */
    void simplifyFunctionTryCatch();
 
    /**
     * Simplify templates
     */
    void simplifyTemplates();
 
    void simplifyDoublePlusAndDoubleMinus();
 
    void simplifyRedundantConsecutiveBraces();
 
    void simplifyArrayAccessSyntax();
 
    void simplifyParameterVoid();
 
    void fillTypeSizes();
 
    void combineOperators();
 
    void combineStringAndCharLiterals();
 
    void concatenateNegativeNumberAndAnyPositive();
 
    void simplifyExternC();
 
    void simplifyRoundCurlyParentheses();
 
    void simplifyTypeIntrinsics();
 
    void simplifySQL();
 
    void checkForEnumsWithTypedef();
 
    void findComplicatedSyntaxErrorsInTemplates();
 
    /**
     * Modify strings in the token list by replacing hex and oct
     * values. E.g. "\x61" -> "a" and "\000" -> "\0"
     * @param source The string to be modified, e.g. "\x61"
     * @return Modified string, e.g. "a"
     */
    static std::string simplifyString(const std::string &source);
 
public:
    /**
     * is token pointing at function head?
     * @param tok         A '(' or ')' token in a possible function head
     * @param endsWith    string after function head
     * @return token matching with endsWith if syntax seems to be a function head else nullptr
     */
    static const Token * isFunctionHead(const Token *tok, const std::string &endsWith);
 
    bool hasIfdef(const Token *start, const Token *end) const;
 
    bool isPacked(const Token * bodyStart) const;
 
private:
 
    /** Simplify pointer to standard type (C only) */
    void simplifyPointerToStandardType();
 
    /** Simplify function pointers */
    void simplifyFunctionPointers();
 
    /**
     * Send error message to error logger about internal bug.
     * @param tok the token that this bug concerns.
     */
    NORETURN void cppcheckError(const Token *tok) const;
 
    /**
     * Setup links for tokens so that one can call Token::link().
     */
    void createLinks();
 
    /**
     * Setup links between < and >.
     */
    void createLinks2();
 
    /**
     * Set isCast() for C++ casts
     */
    void markCppCasts();
 
public:
 
    /** Syntax error */
    NORETURN void syntaxError(const Token *tok, const std::string &code = emptyString) const;
 
    /** Syntax error. Unmatched character. */
    NORETURN void unmatchedToken(const Token *tok) const;
 
    /** Syntax error. C++ code in C file. */
    NORETURN void syntaxErrorC(const Token *tok, const std::string &what) const;
 
    /** Warn about unknown macro(s), configuration is recommended */
    NORETURN void unknownMacroError(const Token *tok1) const;
 
    void unhandledCharLiteral(const Token *tok, const std::string& msg) const;
 
private:
 
    /** Report that there is an unhandled "class x y {" code */
    void unhandled_macro_class_x_y(const Token *tok) const;
 
    /** Check configuration (unknown macros etc) */
    void checkConfiguration() const;
    void macroWithSemicolonError(const Token *tok, const std::string &macroName) const;
 
    /**
     * Is there C++ code in C file?
     */
    void validateC() const;
 
    /**
     * assert that tokens are ok - used during debugging for example
     * to catch problems in simplifyTokenList1/2.
     */
    void validate() const;
 
    /** Detect unknown macros and throw unknownMacro */
    void reportUnknownMacros() const;
 
    /** Detect garbage code and call syntaxError() if found. */
    void findGarbageCode() const;
 
    /** Detect garbage expression */
    static bool isGarbageExpr(const Token *start, const Token *end, bool allowSemicolon);
 
    /**
     * Remove __declspec()
     */
    void simplifyDeclspec();
 
    /**
     * Remove calling convention
     */
    void simplifyCallingConvention();
 
    /**
     * Remove \__attribute\__ ((?))
     */
    void simplifyAttribute();
 
    /** Get function token for a attribute */
    Token* getAttributeFuncTok(Token* tok, bool gccattr) const;
 
    /**
     * Remove \__cppcheck\__ ((?))
     */
    void simplifyCppcheckAttribute();
 
    /** Simplify c++20 spaceship operator */
    void simplifySpaceshipOperator();
 
    /**
     * Remove keywords "volatile", "inline", "register", and "restrict"
     */
    void simplifyKeyword();
 
    /**
     * Remove __asm
     */
    void simplifyAsm();
 
    /**
     * asm heuristics, Put ^{} statements in asm()
     */
    void simplifyAsm2();
 
    /**
     * Simplify \@&hellip;  (compiler extension)
     */
    void simplifyAt();
 
    /**
     * Simplify bitfields - the field width is removed as we don't use it.
     */
    void simplifyBitfields();
 
    /**
     * Remove unnecessary member qualification
     */
    void removeUnnecessaryQualification();
 
    /**
     * Add std:: in front of std classes, when using namespace std; was given
     */
    void simplifyNamespaceStd();
 
    /**
     * Convert Microsoft memory functions
     * CopyMemory(dst, src, len) -> memcpy(dst, src, len)
     * FillMemory(dst, len, val) -> memset(dst, val, len)
     * MoveMemory(dst, src, len) -> memmove(dst, src, len)
     * ZeroMemory(dst, len) -> memset(dst, 0, len)
     */
    void simplifyMicrosoftMemoryFunctions();
 
    /**
     * Convert Microsoft string functions
     * _tcscpy -> strcpy
     */
    void simplifyMicrosoftStringFunctions();
 
    /**
     * Remove Borland code
     */
    void simplifyBorland();
 
    /**
     * Collapse operator name tokens into single token
     * operator = => operator=
     */
    void simplifyOperatorName();
 
    /** simplify overloaded operators: 'obj(123)' => 'obj . operator() ( 123 )' */
    void simplifyOverloadedOperators();
 
    /**
     * Remove [[attribute]] (C++11, C23) from TokenList
     */
    void simplifyCPPAttribute();
 
    /**
     * Convert namespace aliases
     */
    void simplifyNamespaceAliases();
 
    /**
     * Convert C++17 style nested namespace to older style
     */
    void simplifyNestedNamespace();
 
    /**
     * Simplify coroutines - just put parentheses around arguments for
     * co_* keywords so they can be handled like function calls in data
     * flow.
     */
    void simplifyCoroutines();
 
    /**
     * Prepare ternary operators with parentheses so that the AST can be created
     * */
    void prepareTernaryOpForAST();
 
    /**
     * report error message
     */
    void reportError(const Token* tok, const Severity severity, const std::string& id, const std::string& msg, bool inconclusive = false) const;
    void reportError(const std::list<const Token*>& callstack, Severity severity, const std::string& id, const std::string& msg, bool inconclusive = false) const;
 
    bool duplicateTypedef(Token *&tokPtr, const Token *name, const Token *typeDef) const;
 
    void unsupportedTypedef(const Token *tok) const;
 
    static void setVarIdClassFunction(const std::string &classname,
                                      Token * const startToken,
                                      const Token * const endToken,
                                      const std::map<std::string, nonneg int> &varlist,
                                      std::map<nonneg int, std::map<std::string, nonneg int>>& structMembers,
                                      nonneg int &varId_);
 
    /**
     * Output list of unknown types.
     */
    void printUnknownTypes() const;
 
    /** Find end of SQL (or PL/SQL) block */
    static const Token *findSQLBlockEnd(const Token *tokSQLStart);
 
    static bool operatorEnd(const Token * tok);
 
public:
    const SymbolDatabase *getSymbolDatabase() const {
        return mSymbolDatabase;
    }
    void createSymbolDatabase();
 
    /** print --debug output if debug flags match the simplification:
     * 0=unknown/both simplifications
     * 1=1st simplifications
     * 2=2nd simplifications
     */
    void printDebugOutput(int simplification) const;
 
    void dump(std::ostream &out) const;
 
    Token *deleteInvalidTypedef(Token *typeDef);
 
    /**
     * Get variable count.
     * @return number of variables
     */
    nonneg int varIdCount() const {
        return mVarId;
    }
 
    /**
     * Token list: stores all tokens.
     */
    TokenList list;
    // Implement tokens() as a wrapper for convenience when using the TokenList
    const Token* tokens() const {
        return list.front();
    }
 
    Token* tokens() {
        return list.front();
    }
 
    /**
     * Helper function to check whether number is one (1 or 0.1E+1 or 1E+0) or not?
     * @param s the string to check
     * @return true in case is is one and false otherwise.
     */
    static bool isOneNumber(const std::string &s);
 
    /**
     * Helper function to check for start of function execution scope.
     * Do not use this in checks.  Use the symbol database.
     * @param tok pointer to end parentheses of parameter list
     * @return pointer to start brace of function scope or nullptr if not start.
     */
    static const Token * startOfExecutableScope(const Token * tok);
 
    const Settings &getSettings() const {
        return mSettings;
    }
 
    void calculateScopes();
 
    /** Disable copy constructor */
    Tokenizer(const Tokenizer &) = delete;
 
    /** Disable assignment operator */
    Tokenizer &operator=(const Tokenizer &) = delete;
 
    void setDirectives(std::list<Directive> directives);
 
private:
    const Token *processFunc(const Token *tok2, bool inOperator) const;
    Token *processFunc(Token *tok2, bool inOperator);
 
    /**
     * Get new variable id.
     * @return new variable id
     */
    nonneg int newVarId() {
        return ++mVarId;
    }
 
    /** Set pod types */
    void setPodTypes();
 
    /** settings */
    const Settings & mSettings;
 
    /** errorlogger */
    ErrorLogger& mErrorLogger;
 
    /** Symbol database that all checks etc can use */
    SymbolDatabase* mSymbolDatabase{};
 
    TemplateSimplifier * const mTemplateSimplifier;
 
    /** E.g. "A" for code where "#ifdef A" is true. This is used to
        print additional information in error situations. */
    std::string mConfiguration;
 
    /** sizeof information for known types */
    std::map<std::string, int> mTypeSize;
 
    struct TypedefInfo {
        std::string name;
        std::string filename;
        int lineNumber;
        int column;
        bool used;
    };
    std::vector<TypedefInfo> mTypedefInfo;
 
    std::list<Directive> mDirectives;
 
    /** variable count */
    nonneg int mVarId{};
 
    /** unnamed count "Unnamed0", "Unnamed1", "Unnamed2", ... */
    nonneg int mUnnamedCount{};
 
    /**
     * TimerResults
     */
    TimerResults* mTimerResults{};
};
 
/// @}
 
//---------------------------------------------------------------------------
#endif // tokenizeH