antlr3.py

Go to the documentation of this file.

##
#  @package antlr3
# @brief ANTLR3 runtime package
# 
# This module contains all support classes, which are needed to use recognizers
# generated by ANTLR3.
# 
# @mainpage
# 
# \note Please be warned that the line numbers in the API documentation do not
# match the real locations in the source code of the package. This is an
# unintended artifact of doxygen, which I could only convince to use the
# correct module names by concatenating all files from the package into a single
# module file...
# 
# Here is a little overview over the most commonly used classes provided by
# this runtime:
# 
# @section recognizers Recognizers
# 
# These recognizers are baseclasses for the code which is generated by ANTLR3.
# 
# - BaseRecognizer: Base class with common recognizer functionality.
# - Lexer: Base class for lexers.
# - Parser: Base class for parsers.
# - tree.TreeParser: Base class for %tree parser.
# 
# @section streams Streams
# 
# Each recognizer pulls its input from one of the stream classes below. Streams
# handle stuff like buffering, look-ahead and seeking.
# 
# A character stream is usually the first element in the pipeline of a typical
# ANTLR3 application. It is used as the input for a Lexer.
# 
# - ANTLRStringStream: Reads from a string objects. The input should be a unicode
#   object, or ANTLR3 will have trouble decoding non-ascii data.
# - ANTLRFileStream: Opens a file and read the contents, with optional character
#   decoding.
# - ANTLRInputStream: Reads the date from a file-like object, with optional
#   character decoding.
# 
# A Parser needs a TokenStream as input (which in turn is usually fed by a
# Lexer):
# 
# - CommonTokenStream: A basic and most commonly used TokenStream
#   implementation.
# - TokenRewriteStream: A modification of CommonTokenStream that allows the
#   stream to be altered (by the Parser). See the 'tweak' example for a usecase.
# 
# And tree.TreeParser finally fetches its input from a tree.TreeNodeStream:
# 
# - tree.CommonTreeNodeStream: A basic and most commonly used tree.TreeNodeStream
#   implementation.
#   
# 
# @section tokenstrees Tokens and Trees
# 
# A Lexer emits Token objects which are usually buffered by a TokenStream. A
# Parser can build a Tree, if the output=AST option has been set in the grammar.
# 
# The runtime provides these Token implementations:
# 
# - CommonToken: A basic and most commonly used Token implementation.
# - ClassicToken: A Token object as used in ANTLR 2.x, used to %tree
#   construction.
# 
# Tree objects are wrapper for Token objects.
# 
# - tree.CommonTree: A basic and most commonly used Tree implementation.
# 
# A tree.TreeAdaptor is used by the parser to create tree.Tree objects for the
# input Token objects.
# 
# - tree.CommonTreeAdaptor: A basic and most commonly used tree.TreeAdaptor
# implementation.
# 
# 
# @section Exceptions
# 
# RecognitionException are generated, when a recognizer encounters incorrect
# or unexpected input.
# 
# - RecognitionException
#   - MismatchedRangeException
#   - MismatchedSetException
#     - MismatchedNotSetException
#     .
#   - MismatchedTokenException
#   - MismatchedTreeNodeException
#   - NoViableAltException
#   - EarlyExitException
#   - FailedPredicateException
#   .
# .
# 
# A tree.RewriteCardinalityException is raised, when the parsers hits a
# cardinality mismatch during AST construction. Although this is basically a
# bug in your grammar, it can only be detected at runtime.
# 
# - tree.RewriteCardinalityException
#   - tree.RewriteEarlyExitException
#   - tree.RewriteEmptyStreamException
#   .
# .
# 
# 

# tree.RewriteRuleElementStream
# tree.RewriteRuleSubtreeStream
# tree.RewriteRuleTokenStream
# CharStream
# DFA
# TokenSource

# [The "BSD licence"]
# Copyright (c) 2005-2008 Terence Parr
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

__version__ = '3.1.1'

def version_str_to_tuple(version_str):
    import re
    import sys

    if version_str == 'HEAD':
        return (sys.maxint, sys.maxint, sys.maxint, sys.maxint)

    m = re.match(r'(\d+)\.(\d+)(\.(\d+))?(b(\d+))?', version_str)
    if m is None:
        raise ValueError("Bad version string %r" % version_str)

    major = int(m.group(1))
    minor = int(m.group(2))
    patch = int(m.group(4) or 0)
    beta = int(m.group(6) or sys.maxint)

    return (major, minor, patch, beta)


runtime_version_str = __version__
runtime_version = version_str_to_tuple(runtime_version_str)


from constants import *
from dfa import *
from exceptions import *
from recognizers import *
from streams import *
from tokens import *
"""ANTLR3 exception hierarchy"""


from antlr3.constants import INVALID_TOKEN_TYPE


##
# @brief Raised to signal failed backtrack attempt
class BacktrackingFailed(Exception):

    pass


##
# @brief The root of the ANTLR exception hierarchy.
# 
#     To avoid English-only error messages and to generally make things
#     as flexible as possible, these exceptions are not created with strings,
#     but rather the information necessary to generate an error.  Then
#     the various reporting methods in Parser and Lexer can be overridden
#     to generate a localized error message.  For example, MismatchedToken
#     exceptions are built with the expected token type.
#     So, don't expect getMessage() to return anything.
# 
#     Note that as of Java 1.4, you can access the stack trace, which means
#     that you can compute the complete trace of rules from the start symbol.
#     This gives you considerable context information with which to generate
#     useful error messages.
# 
#     ANTLR generates code that throws exceptions upon recognition error and
#     also generates code to catch these exceptions in each rule.  If you
#     want to quit upon first error, you can turn off the automatic error
#     handling mechanism using rulecatch action, but you still need to
#     override methods mismatch and recoverFromMismatchSet.
#     
#     In general, the recognition exceptions can track where in a grammar a
#     problem occurred and/or what was the expected input.  While the parser
#     knows its state (such as current input symbol and line info) that
#     state can change before the exception is reported so current token index
#     is computed and stored at exception time.  From this info, you can
#     perhaps print an entire line of input not just a single token, for example.
#     Better to just say the recognizer had a problem and then let the parser
#     figure out a fancy report.
#     
#     
class RecognitionException(Exception):

    def __init__(self, input=None):
        Exception.__init__(self)

        # What input stream did the error occur in?
        self.input = None

        # What is index of token/char were we looking at when the error
        # occurred?
        self.index = None

        # The current Token when an error occurred.  Since not all streams
        # can retrieve the ith Token, we have to track the Token object.
        # For parsers.  Even when it's a tree parser, token might be set.
        self.token = None

        # If this is a tree parser exception, node is set to the node with
        # the problem.
        self.node = None

        # The current char when an error occurred. For lexers.
        self.c = None

        # Track the line at which the error occurred in case this is
        # generated from a lexer.  We need to track this since the
        # unexpected char doesn't carry the line info.
        self.line = None

        self.charPositionInLine = None

        # If you are parsing a tree node stream, you will encounter som
        # imaginary nodes w/o line/col info.  We now search backwards looking
        # for most recent token with line/col info, but notify getErrorHeader()
        # that info is approximate.
        self.approximateLineInfo = False

        
        if input is not None:
            self.input = input
            self.index = input.index()

            # late import to avoid cyclic dependencies
            from antlr3.streams import TokenStream, CharStream
            from antlr3.tree import TreeNodeStream

            if isinstance(self.input, TokenStream):
                self.token = self.input.LT(1)
                self.line = self.token.line
                self.charPositionInLine = self.token.charPositionInLine

            if isinstance(self.input, TreeNodeStream):
                self.extractInformationFromTreeNodeStream(self.input)

            else:
                if isinstance(self.input, CharStream):
                    self.c = self.input.LT(1)
                    self.line = self.input.line
                    self.charPositionInLine = self.input.charPositionInLine

                else:
                    self.c = self.input.LA(1)

    def extractInformationFromTreeNodeStream(self, nodes):
        from antlr3.tree import Tree, CommonTree
        from antlr3.tokens import CommonToken
        
        self.node = nodes.LT(1)
        adaptor = nodes.adaptor
        payload = adaptor.getToken(self.node)
        if payload is not None:
            self.token = payload
            if payload.line <= 0:
                # imaginary node; no line/pos info; scan backwards
                i = -1
                priorNode = nodes.LT(i)
                while priorNode is not None:
                    priorPayload = adaptor.getToken(priorNode)
                    if priorPayload is not None and priorPayload.line > 0:
                        # we found the most recent real line / pos info
                        self.line = priorPayload.line
                        self.charPositionInLine = priorPayload.charPositionInLine
                        self.approximateLineInfo = True
                        break
                    
                    i -= 1
                    priorNode = nodes.LT(i)
                    
            else: # node created from real token
                self.line = payload.line
                self.charPositionInLine = payload.charPositionInLine
                
        elif isinstance(self.node, Tree):
            self.line = self.node.line
            self.charPositionInLine = self.node.charPositionInLine
            if isinstance(self.node, CommonTree):
                self.token = self.node.token

        else:
            type = adaptor.getType(self.node)
            text = adaptor.getText(self.node)
            self.token = CommonToken(type=type, text=text)

     
    ##
    # Return the token type or char of the unexpected input element
    def getUnexpectedType(self):

        from antlr3.streams import TokenStream
        from antlr3.tree import TreeNodeStream

        if isinstance(self.input, TokenStream):
            return self.token.type

        elif isinstance(self.input, TreeNodeStream):
            adaptor = self.input.treeAdaptor
            return adaptor.getType(self.node)

        else:
            return self.c

    unexpectedType = property(getUnexpectedType)
    

##
# @brief A mismatched char or Token or tree node.
class MismatchedTokenException(RecognitionException):
    
    def __init__(self, expecting, input):
        RecognitionException.__init__(self, input)
        self.expecting = expecting
        

    def __str__(self):
        #return "MismatchedTokenException("+self.expecting+")"
        return "MismatchedTokenException(%r!=%r)" % (
            self.getUnexpectedType(), self.expecting
            )
    __repr__ = __str__


##
# An extra token while parsing a TokenStream
class UnwantedTokenException(MismatchedTokenException):

    def getUnexpectedToken(self):
        return self.token


    def __str__(self):
        exp = ", expected %s" % self.expecting
        if self.expecting == INVALID_TOKEN_TYPE:
            exp = ""

        if self.token is None:
            return "UnwantedTokenException(found=%s%s)" % (None, exp)

        return "UnwantedTokenException(found=%s%s)" % (self.token.text, exp)
    __repr__ = __str__


##
# 
#     We were expecting a token but it's not found.  The current token
#     is actually what we wanted next.
#     
class MissingTokenException(MismatchedTokenException):

    def __init__(self, expecting, input, inserted):
        MismatchedTokenException.__init__(self, expecting, input)

        self.inserted = inserted


    def getMissingType(self):
        return self.expecting


    def __str__(self):
        if self.inserted is not None and self.token is not None:
            return "MissingTokenException(inserted %r at %r)" % (
                self.inserted, self.token.text)

        if self.token is not None:
            return "MissingTokenException(at %r)" % self.token.text

        return "MissingTokenException"
    __repr__ = __str__


##
# @brief The next token does not match a range of expected types.
class MismatchedRangeException(RecognitionException):

    def __init__(self, a, b, input):
        RecognitionException.__init__(self, input)

        self.a = a
        self.b = b
        

    def __str__(self):
        return "MismatchedRangeException(%r not in [%r..%r])" % (
            self.getUnexpectedType(), self.a, self.b
            )
    __repr__ = __str__
    

##
# @brief The next token does not match a set of expected types.
class MismatchedSetException(RecognitionException):

    def __init__(self, expecting, input):
        RecognitionException.__init__(self, input)

        self.expecting = expecting
        

    def __str__(self):
        return "MismatchedSetException(%r not in %r)" % (
            self.getUnexpectedType(), self.expecting
            )
    __repr__ = __str__


##
# @brief Used for remote debugger deserialization
class MismatchedNotSetException(MismatchedSetException):
    
    def __str__(self):
        return "MismatchedNotSetException(%r!=%r)" % (
            self.getUnexpectedType(), self.expecting
            )
    __repr__ = __str__


##
# @brief Unable to decide which alternative to choose.
class NoViableAltException(RecognitionException):

    def __init__(
        self, grammarDecisionDescription, decisionNumber, stateNumber, input
        ):
        RecognitionException.__init__(self, input)

        self.grammarDecisionDescription = grammarDecisionDescription
        self.decisionNumber = decisionNumber
        self.stateNumber = stateNumber


    def __str__(self):
        return "NoViableAltException(%r!=[%r])" % (
            self.unexpectedType, self.grammarDecisionDescription
            )
    __repr__ = __str__
    

##
# @brief The recognizer did not match anything for a (..)+ loop.
class EarlyExitException(RecognitionException):

    def __init__(self, decisionNumber, input):
        RecognitionException.__init__(self, input)

        self.decisionNumber = decisionNumber


##
# @brief A semantic predicate failed during validation.
# 
#     Validation of predicates
#     occurs when normally parsing the alternative just like matching a token.
#     Disambiguating predicate evaluation occurs when we hoist a predicate into
#     a prediction decision.
#     
class FailedPredicateException(RecognitionException):

    def __init__(self, input, ruleName, predicateText):
        RecognitionException.__init__(self, input)
        
        self.ruleName = ruleName
        self.predicateText = predicateText


    def __str__(self):
        return "FailedPredicateException("+self.ruleName+",{"+self.predicateText+"}?)"
    __repr__ = __str__
    

##
# @brief The next tree mode does not match the expected type.
class MismatchedTreeNodeException(RecognitionException):

    def __init__(self, expecting, input):
        RecognitionException.__init__(self, input)
        
        self.expecting = expecting

    def __str__(self):
        return "MismatchedTreeNodeException(%r!=%r)" % (
            self.getUnexpectedType(), self.expecting
            )
    __repr__ = __str__
"""ANTLR3 runtime package"""


EOF = -1

## All tokens go to the parser (unless skip() is called in that rule)
# on a particular "channel".  The parser tunes to a particular channel
# so that whitespace etc... can go to the parser on a "hidden" channel.
DEFAULT_CHANNEL = 0

## Anything on different channel than DEFAULT_CHANNEL is not parsed
# by parser.
HIDDEN_CHANNEL = 99

# Predefined token types
EOR_TOKEN_TYPE = 1

##
# imaginary tree navigation type; traverse "get child" link
DOWN = 2
##
#imaginary tree navigation type; finish with a child list
UP = 3

MIN_TOKEN_TYPE = UP+1
        
INVALID_TOKEN_TYPE = 0

"""ANTLR3 runtime package"""

"""ANTLR3 runtime package"""


from antlr3.constants import EOF, DEFAULT_CHANNEL, INVALID_TOKEN_TYPE

############################################################################
#
# basic token interface
#
############################################################################

##
# @brief Abstract token baseclass.
class Token(object):

    ##
    # @brief Get the text of the token.
    # 
    #         Using setter/getter methods is deprecated. Use o.text instead.
    #         
    def getText(self):
        raise NotImplementedError
    
    ##
    # @brief Set the text of the token.
    # 
    #         Using setter/getter methods is deprecated. Use o.text instead.
    #         
    def setText(self, text):
        raise NotImplementedError


    ##
    # @brief Get the type of the token.
    # 
    #         Using setter/getter methods is deprecated. Use o.type instead.
    def getType(self):

        raise NotImplementedError
    
    ##
    # @brief Get the type of the token.
    # 
    #         Using setter/getter methods is deprecated. Use o.type instead.
    def setType(self, ttype):

        raise NotImplementedError
    
    
    ##
    # @brief Get the line number on which this token was matched
    # 
    #         Lines are numbered 1..n
    #         
    #         Using setter/getter methods is deprecated. Use o.line instead.
    def getLine(self):

        raise NotImplementedError
    
    ##
    # @brief Set the line number on which this token was matched
    # 
    #         Using setter/getter methods is deprecated. Use o.line instead.
    def setLine(self, line):

        raise NotImplementedError
    
    
    ##
    # @brief Get the column of the tokens first character,
    #         
    #         Columns are numbered 0..n-1
    #         
    #         Using setter/getter methods is deprecated. Use o.charPositionInLine instead.
    def getCharPositionInLine(self):

        raise NotImplementedError
    
    ##
    # @brief Set the column of the tokens first character,
    # 
    #         Using setter/getter methods is deprecated. Use o.charPositionInLine instead.
    def setCharPositionInLine(self, pos):

        raise NotImplementedError
    

    ##
    # @brief Get the channel of the token
    # 
    #         Using setter/getter methods is deprecated. Use o.channel instead.
    def getChannel(self):

        raise NotImplementedError
    
    ##
    # @brief Set the channel of the token
    # 
    #         Using setter/getter methods is deprecated. Use o.channel instead.
    def setChannel(self, channel):

        raise NotImplementedError
    

    ##
    # @brief Get the index in the input stream.
    # 
    #         An index from 0..n-1 of the token object in the input stream.
    #         This must be valid in order to use the ANTLRWorks debugger.
    #         
    #         Using setter/getter methods is deprecated. Use o.index instead.
    def getTokenIndex(self):

        raise NotImplementedError
    
    ##
    # @brief Set the index in the input stream.
    # 
    #         Using setter/getter methods is deprecated. Use o.index instead.
    def setTokenIndex(self, index):

        raise NotImplementedError


    ##
    # @brief From what character stream was this token created.
    # 
    #         You don't have to implement but it's nice to know where a Token
    #         comes from if you have include files etc... on the input.
    def getInputStream(self):

        raise NotImplementedError

    ##
    # @brief From what character stream was this token created.
    # 
    #         You don't have to implement but it's nice to know where a Token
    #         comes from if you have include files etc... on the input.
    def setInputStream(self, input):

        raise NotImplementedError


############################################################################
#
# token implementations
#
# Token
# +- CommonToken
# \- ClassicToken
#
############################################################################

##
# @brief Basic token implementation.
# 
#     This implementation does not copy the text from the input stream upon
#     creation, but keeps start/stop pointers into the stream to avoid
#     unnecessary copy operations.
# 
#     
class CommonToken(Token):
    
    def __init__(self, type=None, channel=DEFAULT_CHANNEL, text=None,
                 input=None, start=None, stop=None, oldToken=None):
        Token.__init__(self)
        
        if oldToken is not None:
            self.type = oldToken.type
            self.line = oldToken.line
            self.charPositionInLine = oldToken.charPositionInLine
            self.channel = oldToken.channel
            self.index = oldToken.index
            self._text = oldToken._text
            if isinstance(oldToken, CommonToken):
                self.input = oldToken.input
                self.start = oldToken.start
                self.stop = oldToken.stop
            
        else:
            self.type = type
            self.input = input
            self.charPositionInLine = -1 # set to invalid position
            self.line = 0
            self.channel = channel
            
            #What token number is this from 0..n-1 tokens; < 0 implies invalid index
            self.index = -1
            
            # We need to be able to change the text once in a while.  If
            # this is non-null, then getText should return this.  Note that
            # start/stop are not affected by changing this.
            self._text = text

            # The char position into the input buffer where this token starts
            self.start = start

            # The char position into the input buffer where this token stops
            # This is the index of the last char, *not* the index after it!
            self.stop = stop


    def getText(self):
        if self._text is not None:
            return self._text

        if self.input is None:
            return None
        
        return self.input.substring(self.start, self.stop)


    ##
    # 
    #         Override the text for this token.  getText() will return this text
    #         rather than pulling from the buffer.  Note that this does not mean
    #         that start/stop indexes are not valid.  It means that that input
    #         was converted to a new string in the token object.
    #   
    def setText(self, text):
        self._text = text

    text = property(getText, setText)


    def getType(self):
        return self.type 

    def setType(self, ttype):
        self.type = ttype

    
    def getLine(self):
        return self.line
    
    def setLine(self, line):
        self.line = line


    def getCharPositionInLine(self):
        return self.charPositionInLine
    
    def setCharPositionInLine(self, pos):
        self.charPositionInLine = pos


    def getChannel(self):
        return self.channel
    
    def setChannel(self, channel):
        self.channel = channel
    

    def getTokenIndex(self):
        return self.index
    
    def setTokenIndex(self, index):
        self.index = index


    def getInputStream(self):
        return self.input

    def setInputStream(self, input):
        self.input = input


    def __str__(self):
        if self.type == EOF:
            return "<EOF>"

        channelStr = ""
        if self.channel > 0:
            channelStr = ",channel=" + str(self.channel)

        txt = self.text
        if txt is not None:
            txt = txt.replace("\n","\\\\n")
            txt = txt.replace("\r","\\\\r")
            txt = txt.replace("\t","\\\\t")
        else:
            txt = "<no text>"

        return "[@%d,%d:%d=%r,<%d>%s,%d:%d]" % (
            self.index,
            self.start, self.stop,
            txt,
            self.type, channelStr,
            self.line, self.charPositionInLine
            )
    

##
# @brief Alternative token implementation.
#     
#     A Token object like we'd use in ANTLR 2.x; has an actual string created
#     and associated with this object.  These objects are needed for imaginary
#     tree nodes that have payload objects.  We need to create a Token object
#     that has a string; the tree node will point at this token.  CommonToken
#     has indexes into a char stream and hence cannot be used to introduce
#     new strings.
#     
class ClassicToken(Token):

    def __init__(self, type=None, text=None, channel=DEFAULT_CHANNEL,
                 oldToken=None
                 ):
        Token.__init__(self)
        
        if oldToken is not None:
            self.text = oldToken.text
            self.type = oldToken.type
            self.line = oldToken.line
            self.charPositionInLine = oldToken.charPositionInLine
            self.channel = oldToken.channel
            
        self.text = text
        self.type = type
        self.line = None
        self.charPositionInLine = None
        self.channel = channel
        self.index = None


    def getText(self):
        return self.text

    def setText(self, text):
        self.text = text


    def getType(self):
        return self.type 

    def setType(self, ttype):
        self.type = ttype

    
    def getLine(self):
        return self.line
    
    def setLine(self, line):
        self.line = line


    def getCharPositionInLine(self):
        return self.charPositionInLine
    
    def setCharPositionInLine(self, pos):
        self.charPositionInLine = pos


    def getChannel(self):
        return self.channel
    
    def setChannel(self, channel):
        self.channel = channel
    

    def getTokenIndex(self):
        return self.index
    
    def setTokenIndex(self, index):
        self.index = index


    def getInputStream(self):
        return None

    def setInputStream(self, input):
        pass


    def toString(self):
        channelStr = ""
        if self.channel > 0:
            channelStr = ",channel=" + str(self.channel)
            
        txt = self.text
        if txt is None:
            txt = "<no text>"

        return "[@%r,%r,<%r>%s,%r:%r]" % (self.index,
                                          txt,
                                          self.type,
                                          channelStr,
                                          self.line,
                                          self.charPositionInLine
                                          )
    

    __str__ = toString
    __repr__ = toString



EOF_TOKEN = CommonToken(type=EOF)
        
INVALID_TOKEN = CommonToken(type=INVALID_TOKEN_TYPE)

# In an action, a lexer rule can set token to this SKIP_TOKEN and ANTLR
# will avoid creating a token for this symbol and try to fetch another.
SKIP_TOKEN = CommonToken(type=INVALID_TOKEN_TYPE)


"""ANTLR3 runtime package"""


import codecs
from StringIO import StringIO

from antlr3.constants import DEFAULT_CHANNEL, EOF
from antlr3.tokens import Token, EOF_TOKEN


############################################################################
#
# basic interfaces
#   IntStream
#    +- CharStream
#    \- TokenStream
#
# subclasses must implemented all methods
#
############################################################################

##
# 
#     @brief Base interface for streams of integer values.
# 
#     A simple stream of integers used when all I care about is the char
#     or token type sequence (such as interpretation).
#     
class IntStream(object):

    def consume(self):
        raise NotImplementedError
    

    ##
    # Get int at current input pointer + i ahead where i=1 is next int.
    # 
    #         Negative indexes are allowed.  LA(-1) is previous token (token
    #   just matched).  LA(-i) where i is before first token should
    #   yield -1, invalid char / EOF.
    #   
    def LA(self, i):
        
        raise NotImplementedError
        

    ##
    # 
    #         Tell the stream to start buffering if it hasn't already.  Return
    #         current input position, index(), or some other marker so that
    #         when passed to rewind() you get back to the same spot.
    #         rewind(mark()) should not affect the input cursor.  The Lexer
    #         track line/col info as well as input index so its markers are
    #         not pure input indexes.  Same for tree node streams.
    #         
    def mark(self):

        raise NotImplementedError


    ##
    # 
    #         Return the current input symbol index 0..n where n indicates the
    #         last symbol has been read.  The index is the symbol about to be
    #         read not the most recently read symbol.
    #         
    def index(self):

        raise NotImplementedError


    ##
    # 
    #         Reset the stream so that next call to index would return marker.
    #         The marker will usually be index() but it doesn't have to be.  It's
    #         just a marker to indicate what state the stream was in.  This is
    #         essentially calling release() and seek().  If there are markers
    #         created after this marker argument, this routine must unroll them
    #         like a stack.  Assume the state the stream was in when this marker
    #         was created.
    # 
    #         If marker is None:
    #         Rewind to the input position of the last marker.
    #         Used currently only after a cyclic DFA and just
    #         before starting a sem/syn predicate to get the
    #         input position back to the start of the decision.
    #         Do not "pop" the marker off the state.  mark(i)
    #         and rewind(i) should balance still. It is
    #         like invoking rewind(last marker) but it should not "pop"
    #         the marker off.  It's like seek(last marker's input position).       
    #   
    def rewind(self, marker=None):

        raise NotImplementedError


    ##
    # 
    #         You may want to commit to a backtrack but don't want to force the
    #         stream to keep bookkeeping objects around for a marker that is
    #         no longer necessary.  This will have the same behavior as
    #         rewind() except it releases resources without the backward seek.
    #         This must throw away resources for all markers back to the marker
    #         argument.  So if you're nested 5 levels of mark(), and then release(2)
    #         you have to release resources for depths 2..5.
    #   
    def release(self, marker=None):

        raise NotImplementedError


    ##
    # 
    #         Set the input cursor to the position indicated by index.  This is
    #         normally used to seek ahead in the input stream.  No buffering is
    #         required to do this unless you know your stream will use seek to
    #         move backwards such as when backtracking.
    # 
    #         This is different from rewind in its multi-directional
    #         requirement and in that its argument is strictly an input cursor
    #         (index).
    # 
    #         For char streams, seeking forward must update the stream state such
    #         as line number.  For seeking backwards, you will be presumably
    #         backtracking using the mark/rewind mechanism that restores state and
    #         so this method does not need to update state when seeking backwards.
    # 
    #         Currently, this method is only used for efficient backtracking using
    #         memoization, but in the future it may be used for incremental parsing.
    # 
    #         The index is 0..n-1.  A seek to position i means that LA(1) will
    #         return the ith symbol.  So, seeking to 0 means LA(1) will return the
    #         first element in the stream. 
    #         
    def seek(self, index):

        raise NotImplementedError


    ##
    # 
    #         Only makes sense for streams that buffer everything up probably, but
    #         might be useful to display the entire stream or for testing.  This
    #         value includes a single EOF.
    #   
    def size(self):

        raise