diff options
Diffstat (limited to 'runtime/CSharp3/Sources/Antlr3.Runtime/BufferedTokenStream.cs')
| -rw-r--r-- | runtime/CSharp3/Sources/Antlr3.Runtime/BufferedTokenStream.cs | 425 |
1 files changed, 0 insertions, 425 deletions
diff --git a/runtime/CSharp3/Sources/Antlr3.Runtime/BufferedTokenStream.cs b/runtime/CSharp3/Sources/Antlr3.Runtime/BufferedTokenStream.cs deleted file mode 100644 index 517e64a..0000000 --- a/runtime/CSharp3/Sources/Antlr3.Runtime/BufferedTokenStream.cs +++ /dev/null @@ -1,425 +0,0 @@ -/* - * [The "BSD licence"] - * Copyright (c) 2005-2008 Terence Parr - * All rights reserved. - * - * Conversion to C#: - * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -namespace Antlr.Runtime -{ - using System.Collections.Generic; - using CLSCompliant = System.CLSCompliantAttribute; - using IndexOutOfRangeException = System.IndexOutOfRangeException; - using StringBuilder = System.Text.StringBuilder; - - /** Buffer all input tokens but do on-demand fetching of new tokens from - * lexer. Useful when the parser or lexer has to set context/mode info before - * proper lexing of future tokens. The ST template parser needs this, - * for example, because it has to constantly flip back and forth between - * inside/output templates. E.g., <names:{hi, <it>}> has to parse names - * as part of an expression but "hi, <it>" as a nested template. - * - * You can't use this stream if you pass whitespace or other off-channel - * tokens to the parser. The stream can't ignore off-channel tokens. - * (UnbufferedTokenStream is the same way.) - * - * This is not a subclass of UnbufferedTokenStream because I don't want - * to confuse small moving window of tokens it uses for the full buffer. - */ - [System.Serializable] - public class BufferedTokenStream : ITokenStream, ITokenStreamInformation - { - private ITokenSource _tokenSource; - - /** Record every single token pulled from the source so we can reproduce - * chunks of it later. The buffer in LookaheadStream overlaps sometimes - * as its moving window moves through the input. This list captures - * everything so we can access complete input text. - */ - [CLSCompliant(false)] - protected List<IToken> _tokens = new List<IToken>(100); - - /** Track the last mark() call result value for use in rewind(). */ - private int _lastMarker; - - /** The index into the tokens list of the current token (next token - * to consume). tokens[p] should be LT(1). p=-1 indicates need - * to initialize with first token. The ctor doesn't get a token. - * First call to LT(1) or whatever gets the first token and sets p=0; - */ - [CLSCompliant(false)] - protected int _p = -1; - - public BufferedTokenStream() - { - } - - public BufferedTokenStream(ITokenSource tokenSource) - { - this._tokenSource = tokenSource; - } - - public virtual ITokenSource TokenSource - { - get - { - return _tokenSource; - } - set - { - this._tokenSource = value; - _tokens.Clear(); - _p = -1; - } - } - - public virtual int Index - { - get - { - return _p; - } - } - - /// <summary> - /// How deep have we gone? - /// </summary> - public virtual int Range - { - get; - protected set; - } - - public virtual int Count - { - get - { - return _tokens.Count; - } - } - - public virtual string SourceName - { - get - { - return _tokenSource.SourceName; - } - } - - public virtual IToken LastToken - { - get - { - return LB(1); - } - } - - public virtual IToken LastRealToken - { - get - { - int i = 0; - IToken token; - do - { - i++; - token = LB(i); - } while (token != null && token.Line <= 0); - - return token; - } - } - - public virtual int MaxLookBehind - { - get - { - return int.MaxValue; - } - } - - public virtual int Mark() - { - if (_p == -1) - Setup(); - _lastMarker = Index; - return _lastMarker; - } - - public virtual void Release(int marker) - { - // no resources to release - } - - public virtual void Rewind(int marker) - { - Seek(marker); - } - - public virtual void Rewind() - { - Seek(_lastMarker); - } - - public virtual void Reset() - { - _p = 0; - _lastMarker = 0; - } - - public virtual void Seek(int index) - { - _p = index; - } - - /** Move the input pointer to the next incoming token. The stream - * must become active with LT(1) available. consume() simply - * moves the input pointer so that LT(1) points at the next - * input symbol. Consume at least one token. - * - * Walk past any token not on the channel the parser is listening to. - */ - public virtual void Consume() - { - if (_p == -1) - Setup(); - _p++; - Sync(_p); - } - - /** Make sure index i in tokens has a token. */ - protected virtual void Sync(int i) - { - int n = i - _tokens.Count + 1; // how many more elements we need? - if (n > 0) - Fetch(n); - } - - /** add n elements to buffer */ - protected virtual void Fetch(int n) - { - for (int i = 0; i < n; i++) - { - IToken t = TokenSource.NextToken(); - t.TokenIndex = _tokens.Count; - _tokens.Add(t); - if (t.Type == CharStreamConstants.EndOfFile) - break; - } - } - - public virtual IToken Get(int i) - { - if (i < 0 || i >= _tokens.Count) - { - throw new IndexOutOfRangeException("token index " + i + " out of range 0.." + (_tokens.Count - 1)); - } - return _tokens[i]; - } - -#if false // why is this different from GetTokens(start, count) ? - /// <summary> - /// Get all tokens from start..(start+count-1) inclusively - /// </summary> - public virtual List<IToken> Get(int start, int count) - { - if (start < 0) - throw new ArgumentOutOfRangeException("start"); - if (count < 0) - throw new ArgumentOutOfRangeException("count"); - if (start + count >= _tokens.Count) - throw new ArgumentException(); - - if (_p == -1) - Setup(); - - List<IToken> subset = new List<IToken>(count); - for (int i = 0; i < count; i++) - { - IToken token = _tokens[i]; - if (token.Type == TokenTypes.EndOfFile) - break; - - subset.Add(token); - } - - return subset; - } -#endif - - public virtual int LA(int i) - { - return LT(i).Type; - } - - protected virtual IToken LB(int k) - { - if ((_p - k) < 0) - return null; - - return _tokens[_p - k]; - } - - public virtual IToken LT(int k) - { - if (_p == -1) - Setup(); - if (k == 0) - return null; - if (k < 0) - return LB(-k); - - int i = _p + k - 1; - Sync(i); - if (i >= _tokens.Count) - { - // EOF must be last token - return _tokens[_tokens.Count - 1]; - } - - if (i > Range) - Range = i; - - return _tokens[_p + k - 1]; - } - - protected virtual void Setup() - { - Sync(0); - _p = 0; - } - - public virtual List<IToken> GetTokens() - { - return _tokens; - } - - public virtual List<IToken> GetTokens(int start, int stop) - { - return GetTokens(start, stop, default(BitSet)); - } - - /** Given a start and stop index, return a List of all tokens in - * the token type BitSet. Return null if no tokens were found. This - * method looks at both on and off channel tokens. - */ - public virtual List<IToken> GetTokens(int start, int stop, BitSet types) - { - if (_p == -1) - Setup(); - if (stop >= _tokens.Count) - stop = _tokens.Count - 1; - if (start < 0) - start = 0; - if (start > stop) - return null; - - // list = tokens[start:stop]:{Token t, t.getType() in types} - List<IToken> filteredTokens = new List<IToken>(); - for (int i = start; i <= stop; i++) - { - IToken t = _tokens[i]; - if (types == null || types.Member(t.Type)) - { - filteredTokens.Add(t); - } - } - if (filteredTokens.Count == 0) - { - filteredTokens = null; - } - return filteredTokens; - } - - public virtual List<IToken> GetTokens(int start, int stop, IEnumerable<int> types) - { - return GetTokens(start, stop, new BitSet(types)); - } - - public virtual List<IToken> GetTokens(int start, int stop, int ttype) - { - return GetTokens(start, stop, BitSet.Of(ttype)); - } - - public override string ToString() - { - if (_p == -1) - Setup(); - - Fill(); - return ToString(0, _tokens.Count - 1); - } - - public virtual string ToString(int start, int stop) - { - if (start < 0 || stop < 0) - return null; - if (_p == -1) - Setup(); - if (stop >= _tokens.Count) - stop = _tokens.Count - 1; - - StringBuilder buf = new StringBuilder(); - for (int i = start; i <= stop; i++) - { - IToken t = _tokens[i]; - if (t.Type == CharStreamConstants.EndOfFile) - break; - buf.Append(t.Text); - } - - return buf.ToString(); - } - - public virtual string ToString(IToken start, IToken stop) - { - if (start != null && stop != null) - { - return ToString(start.TokenIndex, stop.TokenIndex); - } - return null; - } - - public virtual void Fill() - { - if (_p == -1) - Setup(); - - if (_tokens[_p].Type == CharStreamConstants.EndOfFile) - return; - - int i = _p + 1; - Sync(i); - while (_tokens[i].Type != CharStreamConstants.EndOfFile) - { - i++; - Sync(i); - } - } - } -} |