tokenizer.h

Go to the documentation of this file.

/*
 * This file is part of the Code::Blocks IDE and licensed under the GNU General Public License, version 3
 * http://www.gnu.org/licenses/gpl-3.0.html
 */

#ifndef TOKENIZER_H
#define TOKENIZER_H

#include <wx/string.h>
#include <configmanager.h>
#include <filemanager.h>
#include "token.h"

#include <stack>
#include <list>


enum TokenizerState
{
    tsNormal        = 0x0000,
    tsRawExpression = 0x0001
};

enum PreprocessorType
{
    ptIf                = 0x0001,   
    ptIfdef             = 0x0002,   
    ptIfndef            = 0x0003,   
    ptElif              = 0x0004,   
    ptElifdef           = 0x0005,   
    ptElifndef          = 0x0006,   
    ptElse              = 0x0007,   
    ptEndif             = 0x0008,   
    ptDefine            = 0x0009,   
    ptUndef             = 0x000A,   
    ptOthers            = 0x000B    
};

struct TokenizerOptions
{
    bool wantPreprocessor;
    bool storeDocumentation;
};

class Tokenizer
{
public:
    Tokenizer(TokenTree* tokenTree, const wxString& filename = wxEmptyString);

    ~Tokenizer();

    bool Init(const wxString& filename = wxEmptyString, LoaderBase* loader = 0);

    bool InitFromBuffer(const wxString& buffer, const wxString& fileOfBuffer = wxEmptyString,
                        size_t initLineNumber = 0);

    wxString GetToken();

    wxString PeekToken();

    void     UngetToken();

    void SetTokenizerOption(bool wantPreprocessor, bool storeDocumentation)
    {
        m_TokenizerOptions.wantPreprocessor = wantPreprocessor;
        m_TokenizerOptions.storeDocumentation = storeDocumentation;
    };

    void SetState(TokenizerState state)
    {
        m_State = state;
    };

    TokenizerState GetState()
    {
        return m_State;
    }

    const wxString& GetFilename() const
    {
        return m_Filename;
    };

    unsigned int GetLineNumber() const
    {
        return m_LineNumber;
    };

    unsigned int GetNestingLevel() const
    {
        return m_NestLevel;
    };

    void SaveNestingLevel()
    {
        m_SavedNestingLevel = m_NestLevel;
    };

    void RestoreNestingLevel()
    {
        m_NestLevel = m_SavedNestingLevel;
    };

    bool IsOK() const
    {
        return m_IsOK;
    };

    wxString ReadToEOL(bool stripUnneeded = true);

    void ReadParentheses(wxString& str);

    bool SkipToEOL();

    bool SkipToInlineCommentEnd();

    bool IsEOF() const
    {
        return m_TokenIndex >= m_BufferLen;
    }

    bool NotEOF() const
    {
        return m_TokenIndex < m_BufferLen;
    }

    bool ReplaceBufferText(const wxString& target, const Token* macro = 0);

    bool ReplaceMacroUsage(const Token* tk);

    int GetFirstTokenPosition(const wxString& buffer, const wxString& target)
    {
        return GetFirstTokenPosition(buffer.GetData(), buffer.Len(), target.GetData(), target.Len());
    }

    int GetFirstTokenPosition(const wxChar* buffer, const size_t bufferLen,
                              const wxChar* key, const size_t keyLen);

    int KMP_Find(const wxChar* text, const wxChar* pattern, const int patternLen);

    void SetLastTokenIdx(int tokenIdx);

protected:
    void BaseInit();

    wxString DoGetToken();

    bool CheckMacroUsageAndReplace();

    bool Lex();

    bool ReadFile();

    bool IsEscapedChar();

    bool SkipToChar(const wxChar& ch);

    bool SkipUnwanted();

    bool SkipWhiteSpace();

    bool SkipComment();

    bool SkipPreprocessorBranch();

    bool SkipString();

    bool SkipToStringEnd(const wxChar& ch);

    bool MoveToNextChar();

    wxChar CurrentChar() const
    {
        if(m_TokenIndex < m_BufferLen)
            return m_Buffer.GetChar(m_TokenIndex);
        return 0;
    };

    wxChar CurrentCharMoveNext()
    {
        wxChar c = CurrentChar();
        m_TokenIndex++;
        return c;
    };

    wxChar NextChar() const
    {
        if ((m_TokenIndex + 1) >= m_BufferLen) // m_TokenIndex + 1) < 0  can never be true
            return 0;

        return m_Buffer.GetChar(m_TokenIndex + 1);
    };

    wxChar PreviousChar() const
    {
        if (m_TokenIndex > 0 && m_BufferLen > 0) // m_TokenIndex > m_BufferLen can never be true
            return m_Buffer.GetChar(m_TokenIndex - 1);

        return 0;
    };

private:
    inline bool CharInString(const wxChar ch, const wxChar* chars) const
    {
        int len = wxStrlen(chars);
        for (int i = 0; i < len; ++i)
        {
            if (ch == chars[i])
                return true;
        }
        return false;
    };

    inline bool IsBackslashBeforeEOL()
    {
        wxChar last = PreviousChar();
        // if DOS line endings, we have hit \r and we skip to \n...
        if (last == _T('\r') && m_TokenIndex >= 2)
            return m_Buffer.GetChar(m_TokenIndex - 2) == _T('\\');
        return last == _T('\\');
    }

    bool CalcConditionExpression();

    bool IsMacroDefined();

    void HandleDefines();

    void HandleUndefs();

    void AddMacroDefinition(wxString name, int line, wxString para, wxString substitues);

    void SkipToNextConditionPreprocessor();

    void SkipToEndConditionPreprocessor();

    PreprocessorType GetPreprocessorType();

    void HandleConditionPreprocessor(const PreprocessorType type);

    bool SplitArguments(wxArrayString& results);

    bool GetMacroExpandedText(const Token* tk, wxString& expandedText);

    void KMP_GetNextVal(const wxChar* pattern, int next[]);

    TokenizerOptions     m_TokenizerOptions;

    TokenTree*           m_TokenTree;

    wxString             m_Filename;
    unsigned int         m_FileIdx;
    wxString             m_Buffer;
    unsigned int         m_BufferLen;

    wxString             m_Lex;

    wxString             m_Token;                

    unsigned int         m_TokenIndex;
    unsigned int         m_LineNumber;
    unsigned int         m_NestLevel;

    unsigned int         m_UndoTokenIndex;
    unsigned int         m_UndoLineNumber;
    unsigned int         m_UndoNestLevel;

    bool                 m_PeekAvailable;
    wxString             m_PeekToken;
    unsigned int         m_PeekTokenIndex;
    unsigned int         m_PeekLineNumber;
    unsigned int         m_PeekNestLevel;

    unsigned int         m_SavedTokenIndex;
    unsigned int         m_SavedLineNumber;
    unsigned int         m_SavedNestingLevel;

    bool                 m_IsOK;
    TokenizerState       m_State;
    LoaderBase*          m_Loader;
    std::stack<bool>     m_ExpressionResult;


    struct ExpandedMacro
    {
        ExpandedMacro():m_Macro(0)
        {
        };
        unsigned int m_Begin;
        unsigned int m_End;
        const Token* m_Macro;
    };

    std::list<ExpandedMacro>    m_ExpandedMacros;

    wxString             m_NextTokenDoc;

    int                  m_LastTokenIdx;

    bool m_ReadingMacroDefinition;
};

#endif // TOKENIZER_H