diff options
Diffstat (limited to 'runtime/CSharp3/Sources/Antlr3.Runtime/BufferedTokenStream.cs')
| -rw-r--r-- | runtime/CSharp3/Sources/Antlr3.Runtime/BufferedTokenStream.cs | 425 |
1 files changed, 425 insertions, 0 deletions
diff --git a/runtime/CSharp3/Sources/Antlr3.Runtime/BufferedTokenStream.cs b/runtime/CSharp3/Sources/Antlr3.Runtime/BufferedTokenStream.cs new file mode 100644 index 0000000..517e64a --- /dev/null +++ b/runtime/CSharp3/Sources/Antlr3.Runtime/BufferedTokenStream.cs @@ -0,0 +1,425 @@ +/* + * [The "BSD licence"] + * Copyright (c) 2005-2008 Terence Parr + * All rights reserved. + * + * Conversion to C#: + * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +namespace Antlr.Runtime +{ + using System.Collections.Generic; + using CLSCompliant = System.CLSCompliantAttribute; + using IndexOutOfRangeException = System.IndexOutOfRangeException; + using StringBuilder = System.Text.StringBuilder; + + /** Buffer all input tokens but do on-demand fetching of new tokens from + * lexer. Useful when the parser or lexer has to set context/mode info before + * proper lexing of future tokens. The ST template parser needs this, + * for example, because it has to constantly flip back and forth between + * inside/output templates. E.g., <names:{hi, <it>}> has to parse names + * as part of an expression but "hi, <it>" as a nested template. + * + * You can't use this stream if you pass whitespace or other off-channel + * tokens to the parser. The stream can't ignore off-channel tokens. + * (UnbufferedTokenStream is the same way.) + * + * This is not a subclass of UnbufferedTokenStream because I don't want + * to confuse small moving window of tokens it uses for the full buffer. + */ + [System.Serializable] + public class BufferedTokenStream : ITokenStream, ITokenStreamInformation + { + private ITokenSource _tokenSource; + + /** Record every single token pulled from the source so we can reproduce + * chunks of it later. The buffer in LookaheadStream overlaps sometimes + * as its moving window moves through the input. This list captures + * everything so we can access complete input text. + */ + [CLSCompliant(false)] + protected List<IToken> _tokens = new List<IToken>(100); + + /** Track the last mark() call result value for use in rewind(). */ + private int _lastMarker; + + /** The index into the tokens list of the current token (next token + * to consume). tokens[p] should be LT(1). p=-1 indicates need + * to initialize with first token. The ctor doesn't get a token. + * First call to LT(1) or whatever gets the first token and sets p=0; + */ + [CLSCompliant(false)] + protected int _p = -1; + + public BufferedTokenStream() + { + } + + public BufferedTokenStream(ITokenSource tokenSource) + { + this._tokenSource = tokenSource; + } + + public virtual ITokenSource TokenSource + { + get + { + return _tokenSource; + } + set + { + this._tokenSource = value; + _tokens.Clear(); + _p = -1; + } + } + + public virtual int Index + { + get + { + return _p; + } + } + + /// <summary> + /// How deep have we gone? + /// </summary> + public virtual int Range + { + get; + protected set; + } + + public virtual int Count + { + get + { + return _tokens.Count; + } + } + + public virtual string SourceName + { + get + { + return _tokenSource.SourceName; + } + } + + public virtual IToken LastToken + { + get + { + return LB(1); + } + } + + public virtual IToken LastRealToken + { + get + { + int i = 0; + IToken token; + do + { + i++; + token = LB(i); + } while (token != null && token.Line <= 0); + + return token; + } + } + + public virtual int MaxLookBehind + { + get + { + return int.MaxValue; + } + } + + public virtual int Mark() + { + if (_p == -1) + Setup(); + _lastMarker = Index; + return _lastMarker; + } + + public virtual void Release(int marker) + { + // no resources to release + } + + public virtual void Rewind(int marker) + { + Seek(marker); + } + + public virtual void Rewind() + { + Seek(_lastMarker); + } + + public virtual void Reset() + { + _p = 0; + _lastMarker = 0; + } + + public virtual void Seek(int index) + { + _p = index; + } + + /** Move the input pointer to the next incoming token. The stream + * must become active with LT(1) available. consume() simply + * moves the input pointer so that LT(1) points at the next + * input symbol. Consume at least one token. + * + * Walk past any token not on the channel the parser is listening to. + */ + public virtual void Consume() + { + if (_p == -1) + Setup(); + _p++; + Sync(_p); + } + + /** Make sure index i in tokens has a token. */ + protected virtual void Sync(int i) + { + int n = i - _tokens.Count + 1; // how many more elements we need? + if (n > 0) + Fetch(n); + } + + /** add n elements to buffer */ + protected virtual void Fetch(int n) + { + for (int i = 0; i < n; i++) + { + IToken t = TokenSource.NextToken(); + t.TokenIndex = _tokens.Count; + _tokens.Add(t); + if (t.Type == CharStreamConstants.EndOfFile) + break; + } + } + + public virtual IToken Get(int i) + { + if (i < 0 || i >= _tokens.Count) + { + throw new IndexOutOfRangeException("token index " + i + " out of range 0.." + (_tokens.Count - 1)); + } + return _tokens[i]; + } + +#if false // why is this different from GetTokens(start, count) ? + /// <summary> + /// Get all tokens from start..(start+count-1) inclusively + /// </summary> + public virtual List<IToken> Get(int start, int count) + { + if (start < 0) + throw new ArgumentOutOfRangeException("start"); + if (count < 0) + throw new ArgumentOutOfRangeException("count"); + if (start + count >= _tokens.Count) + throw new ArgumentException(); + + if (_p == -1) + Setup(); + + List<IToken> subset = new List<IToken>(count); + for (int i = 0; i < count; i++) + { + IToken token = _tokens[i]; + if (token.Type == TokenTypes.EndOfFile) + break; + + subset.Add(token); + } + + return subset; + } +#endif + + public virtual int LA(int i) + { + return LT(i).Type; + } + + protected virtual IToken LB(int k) + { + if ((_p - k) < 0) + return null; + + return _tokens[_p - k]; + } + + public virtual IToken LT(int k) + { + if (_p == -1) + Setup(); + if (k == 0) + return null; + if (k < 0) + return LB(-k); + + int i = _p + k - 1; + Sync(i); + if (i >= _tokens.Count) + { + // EOF must be last token + return _tokens[_tokens.Count - 1]; + } + + if (i > Range) + Range = i; + + return _tokens[_p + k - 1]; + } + + protected virtual void Setup() + { + Sync(0); + _p = 0; + } + + public virtual List<IToken> GetTokens() + { + return _tokens; + } + + public virtual List<IToken> GetTokens(int start, int stop) + { + return GetTokens(start, stop, default(BitSet)); + } + + /** Given a start and stop index, return a List of all tokens in + * the token type BitSet. Return null if no tokens were found. This + * method looks at both on and off channel tokens. + */ + public virtual List<IToken> GetTokens(int start, int stop, BitSet types) + { + if (_p == -1) + Setup(); + if (stop >= _tokens.Count) + stop = _tokens.Count - 1; + if (start < 0) + start = 0; + if (start > stop) + return null; + + // list = tokens[start:stop]:{Token t, t.getType() in types} + List<IToken> filteredTokens = new List<IToken>(); + for (int i = start; i <= stop; i++) + { + IToken t = _tokens[i]; + if (types == null || types.Member(t.Type)) + { + filteredTokens.Add(t); + } + } + if (filteredTokens.Count == 0) + { + filteredTokens = null; + } + return filteredTokens; + } + + public virtual List<IToken> GetTokens(int start, int stop, IEnumerable<int> types) + { + return GetTokens(start, stop, new BitSet(types)); + } + + public virtual List<IToken> GetTokens(int start, int stop, int ttype) + { + return GetTokens(start, stop, BitSet.Of(ttype)); + } + + public override string ToString() + { + if (_p == -1) + Setup(); + + Fill(); + return ToString(0, _tokens.Count - 1); + } + + public virtual string ToString(int start, int stop) + { + if (start < 0 || stop < 0) + return null; + if (_p == -1) + Setup(); + if (stop >= _tokens.Count) + stop = _tokens.Count - 1; + + StringBuilder buf = new StringBuilder(); + for (int i = start; i <= stop; i++) + { + IToken t = _tokens[i]; + if (t.Type == CharStreamConstants.EndOfFile) + break; + buf.Append(t.Text); + } + + return buf.ToString(); + } + + public virtual string ToString(IToken start, IToken stop) + { + if (start != null && stop != null) + { + return ToString(start.TokenIndex, stop.TokenIndex); + } + return null; + } + + public virtual void Fill() + { + if (_p == -1) + Setup(); + + if (_tokens[_p].Type == CharStreamConstants.EndOfFile) + return; + + int i = _p + 1; + Sync(i); + while (_tokens[i].Type != CharStreamConstants.EndOfFile) + { + i++; + Sync(i); + } + } + } +} |