scripttokeniser.h

Go to the documentation of this file.

/*
 Copyright (C) 2001-2006, William Joseph.
 All Rights Reserved.

 This file is part of GtkRadiant.

 GtkRadiant is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2 of the License, or
 (at your option) any later version.

 GtkRadiant is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with GtkRadiant; if not, write to the Free Software
 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

#if !defined(INCLUDED_SCRIPT_SCRIPTTOKENISER_H)
#define INCLUDED_SCRIPT_SCRIPTTOKENISER_H

#include "iscriplib.h"

class ScriptTokeniser: public Tokeniser
{
        enum CharType
        {
            eWhitespace, eCharToken, eNewline, eCharQuote, eCharSolidus, eCharStar, eCharSpecial,
        };

        typedef bool (ScriptTokeniser::*Tokenise) (char c);

        Tokenise m_stack[3];
        Tokenise* m_state;
        SingleCharacterInputStream<TextInputStream> m_istream;
        std::size_t m_scriptline;
        std::size_t m_scriptcolumn;

        char m_token[MAXTOKEN];
        char* m_write;

        char m_current;
        bool m_eof;
        bool m_unget;
        bool m_emit;

        bool m_special;

        CharType charType (const char c)
        {
            switch (c) {
            case '\n':
                return eNewline;
            case '"':
                return eCharQuote;
            case '/':
                return eCharSolidus;
            case '*':
                return eCharStar;
            case '{':
            case '(':
            case '}':
            case ')':
            case '[':
            case ']':
            case ',':
            case ':':
                return (m_special) ? eCharSpecial : eCharToken;
            }

            if (c > 32) {
                return eCharToken;
            }
            return eWhitespace;
        }

        Tokenise state ()
        {
            return *m_state;
        }
        void push (Tokenise state)
        {
            ASSERT_MESSAGE(m_state != m_stack + 2, "token parser: illegal stack push");
            *(++m_state) = state;
        }
        void pop ()
        {
            ASSERT_MESSAGE(m_state != m_stack, "token parser: illegal stack pop");
            --m_state;
        }
        void add (const char c)
        {
            if (m_write < m_token + MAXTOKEN - 1) {
                *m_write++ = c;
            }
        }
        void remove ()
        {
            ASSERT_MESSAGE(m_write > m_token, "no char to remove");
            --m_write;
        }

        bool tokeniseDefault (char c)
        {
            switch (charType(c)) {
            case eNewline:
                break;
            case eCharToken:
            case eCharStar:
                push(Tokenise(&ScriptTokeniser::tokeniseToken));
                add(c);
                break;
            case eCharSpecial:
                push(Tokenise(&ScriptTokeniser::tokeniseSpecial));
                add(c);
                break;
            case eCharQuote:
                push(Tokenise(&ScriptTokeniser::tokeniseQuotedToken));
                break;
            case eCharSolidus:
                push(Tokenise(&ScriptTokeniser::tokeniseSolidus));
                break;
            default:
                break;
            }
            return true;
        }
        bool tokeniseToken (char c)
        {
            switch (charType(c)) {
            case eNewline:
            case eWhitespace:
            case eCharQuote:
            case eCharSpecial:
                pop();
                m_emit = true; // emit token
                break;
            case eCharSolidus:
            case eCharToken:
            case eCharStar:
                add(c);
                break;
            default:
                break;
            }
            return true;
        }
        bool tokeniseQuotedToken (char c)
        {
            switch (charType(c)) {
            case eNewline:
                break;
            case eWhitespace:
            case eCharToken:
            case eCharSolidus:
            case eCharStar:
            case eCharSpecial:
                add(c);
                break;
            case eCharQuote:
                pop();
                push(Tokenise(&ScriptTokeniser::tokeniseEndQuote));
                break;
            default:
                break;
            }
            return true;
        }
        bool tokeniseSolidus (char c)
        {
            switch (charType(c)) {
            case eNewline:
            case eWhitespace:
            case eCharQuote:
            case eCharSpecial:
                pop();
                add('/');
                m_emit = true; // emit single slash
                break;
            case eCharToken:
                pop();
                add('/');
                add(c);
                break;
            case eCharSolidus:
                pop();
                push(Tokenise(&ScriptTokeniser::tokeniseComment));
                break; // don't emit single slash
            case eCharStar:
                pop();
                push(Tokenise(&ScriptTokeniser::tokeniseBlockComment));
                break; // don't emit single slash
            default:
                break;
            }
            return true;
        }
        bool tokeniseComment (char c)
        {
            if (c == '\n') {
                pop();
                if (state() == Tokenise(&ScriptTokeniser::tokeniseToken)) {
                    pop();
                    m_emit = true; // emit token immediately preceding comment
                }
            }
            return true;
        }
        bool tokeniseBlockComment (char c)
        {
            if (c == '*') {
                pop();
                push(Tokenise(&ScriptTokeniser::tokeniseEndBlockComment));
            }
            return true;
        }
        bool tokeniseEndBlockComment (char c)
        {
            switch (c) {
            case '/':
                pop();
                if (state() == Tokenise(&ScriptTokeniser::tokeniseToken)) {
                    pop();
                    m_emit = true; // emit token immediately preceding comment
                }
                break; // don't emit comment
            case '*':
                break; // no state change
            default:
                pop();
                push(Tokenise(&ScriptTokeniser::tokeniseBlockComment));
                break;
            }
            return true;
        }
        bool tokeniseEndQuote (char c)
        {
            pop();
            m_emit = true; // emit quoted token
            return true;
        }
        bool tokeniseSpecial (char c)
        {
            pop();
            m_emit = true; // emit single-character token
            return true;
        }

        bool tokenise ()
        {
            m_write = m_token;
            while (!eof()) {
                char c = m_current;

                if (!((*this).*state())(c)) {
                    // parse error
                    m_eof = true;
                    return false;
                }
                if (m_emit) {
                    m_emit = false;
                    return true;
                }

                if (c == '\n') {
                    ++m_scriptline;
                    m_scriptcolumn = 1;
                } else {
                    ++m_scriptcolumn;
                }

                m_eof = !m_istream.readChar(m_current);
            }
            return m_write != m_token;
        }

        const char* fillToken ()
        {
            if (!tokenise()) {
                return 0;
            }

            add('\0');
            return m_token;
        }

        bool eof ()
        {
            return m_eof;
        }

    public:
        ScriptTokeniser (TextInputStream& istream, bool special) :
            m_state(m_stack), m_istream(istream), m_scriptline(1), m_scriptcolumn(1), m_unget(false), m_emit(false),
                    m_special(special)
        {
            m_stack[0] = Tokenise(&ScriptTokeniser::tokeniseDefault);
            m_eof = !m_istream.readChar(m_current);
            m_token[MAXTOKEN - 1] = '\0';
        }

        const std::string getToken ()
        {
            if (m_unget) {
                m_unget = false;
                return m_token;
            }

            const char *token = fillToken();
            if (token)
                return std::string(token);

            return "";
        }

        void ungetToken ()
        {
            ASSERT_MESSAGE(!m_unget, "can't unget more than one token");
            m_unget = true;
        }
        std::size_t getLine () const
        {
            return m_scriptline;
        }
        std::size_t getColumn () const
        {
            return m_scriptcolumn;
        }
};

inline Tokeniser* NewScriptTokeniser (TextInputStream& istream)
{
    return new ScriptTokeniser(istream, true);
}

inline Tokeniser* NewSimpleTokeniser (TextInputStream& istream)
{
    return new ScriptTokeniser(istream, false);
}

#endif