diff options
Diffstat (limited to 'runtime/CSharp3/Sources/Antlr3.Runtime.Test/SlimParsing/SlimTokenStream.cs')
| -rw-r--r-- | runtime/CSharp3/Sources/Antlr3.Runtime.Test/SlimParsing/SlimTokenStream.cs | 438 |
1 files changed, 438 insertions, 0 deletions
diff --git a/runtime/CSharp3/Sources/Antlr3.Runtime.Test/SlimParsing/SlimTokenStream.cs b/runtime/CSharp3/Sources/Antlr3.Runtime.Test/SlimParsing/SlimTokenStream.cs new file mode 100644 index 0000000..50da1ed --- /dev/null +++ b/runtime/CSharp3/Sources/Antlr3.Runtime.Test/SlimParsing/SlimTokenStream.cs @@ -0,0 +1,438 @@ +/* + * [The "BSD license"] + * Copyright (c) 2011 Terence Parr + * All rights reserved. + * + * Conversion to C#: + * Copyright (c) 2011 Sam Harwell, Pixel Mine, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +namespace Antlr.Runtime +{ + using System.Collections.Generic; + using System; + using System.Text; + + /** <summary> + * A pretty quick CharStream that pulls all data from an array + * directly. Every method call counts in the lexer. Java's + * strings aren't very good so I'm avoiding. + * </summary> + */ + [System.Serializable] + public class FastTokenStream + : ITokenStream<SlimToken> + { + [System.NonSerialized] + ITokenSource<SlimToken> _tokenSource; + + /** <summary> + * Record every single token pulled from the source so we can reproduce + * chunks of it later. + * </summary> + */ + protected List<SlimToken> tokens; + + /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */ + protected int channel = TokenChannels.Default; + + /** <summary>Track the last mark() call result value for use in rewind().</summary> */ + protected int lastMarker; + + /** <summary> + * The index into the tokens list of the current token (next token + * to consume). p==-1 indicates that the tokens list is empty + * </summary> + */ + protected int p = -1; + + public FastTokenStream() + { + tokens = new List<SlimToken>( 500 ); + } + + public FastTokenStream( ITokenSource<SlimToken> tokenSource ) + : this() + { + this._tokenSource = tokenSource; + } + + public FastTokenStream( ITokenSource<SlimToken> tokenSource, int channel ) + : this( tokenSource ) + { + this.channel = channel; + } + + public int Index + { + get + { + return p; + } + } + + /** <summary>Reset this token stream by setting its token source.</summary> */ + public void SetTokenSource( ITokenSource<SlimToken> tokenSource ) + { + this._tokenSource = tokenSource; + tokens.Clear(); + p = -1; + channel = TokenChannels.Default; + } + + /** <summary> + * Load all tokens from the token source and put in tokens. + * This is done upon first LT request because you might want to + * set some token type / channel overrides before filling buffer. + * </summary> + */ + public void FillBuffer() + { + // fast return if the buffer is already full + if ( p != -1 ) + return; + + int index = 0; + SlimToken t = _tokenSource.NextToken(); + while ( t.Type != CharStreamConstants.EndOfFile ) + { + //t.TokenIndex = index; + tokens.Add( t ); + index++; + + t = _tokenSource.NextToken(); + } + // leave p pointing at first token on channel + p = 0; + p = SkipOffTokenChannels( p ); + } + + /** <summary> + * Move the input pointer to the next incoming token. The stream + * must become active with LT(1) available. consume() simply + * moves the input pointer so that LT(1) points at the next + * input symbol. Consume at least one token. + * </summary> + * + * <remarks> + * Walk past any token not on the channel the parser is listening to. + * </remarks> + */ + public void Consume() + { + if ( p < tokens.Count ) + { + p++; + p = SkipOffTokenChannels( p ); // leave p on valid token + } + } + + /** <summary>Given a starting index, return the index of the first on-channel token.</summary> */ + protected int SkipOffTokenChannels( int i ) + { + int n = tokens.Count; + while ( i < n && tokens[i].Channel != channel ) + { + i++; + } + return i; + } + + protected int SkipOffTokenChannelsReverse( int i ) + { + while ( i >= 0 && tokens[i].Channel != channel ) + { + i--; + } + return i; + } + + public IList<SlimToken> GetTokens() + { + if ( p == -1 ) + { + FillBuffer(); + } + return tokens; + } + + public IList<SlimToken> GetTokens( int start, int stop ) + { + return GetTokens( start, stop, (BitSet)null ); + } + + /** <summary> + * Given a start and stop index, return a List of all tokens in + * the token type BitSet. Return null if no tokens were found. This + * method looks at both on and off channel tokens. + * </summary> + */ + public IList<SlimToken> GetTokens( int start, int stop, BitSet types ) + { + if ( p == -1 ) + { + FillBuffer(); + } + if ( stop >= tokens.Count ) + { + stop = tokens.Count - 1; + } + if ( start < 0 ) + { + start = 0; + } + if ( start > stop ) + { + return null; + } + + // list = tokens[start:stop]:{Token t, t.getType() in types} + List<SlimToken> filteredTokens = new List<SlimToken>(); + for ( int i = start; i <= stop; i++ ) + { + SlimToken t = tokens[i]; + if ( types == null || types.Member( t.Type ) ) + { + filteredTokens.Add( t ); + } + } + if ( filteredTokens.Count == 0 ) + { + filteredTokens = null; + } + return filteredTokens; + } + + public IList<SlimToken> GetTokens( int start, int stop, IList<int> types ) + { + return GetTokens( start, stop, new BitSet( types ) ); + } + + public IList<SlimToken> GetTokens( int start, int stop, int ttype ) + { + return GetTokens( start, stop, BitSet.Of( ttype ) ); + } + + /** <summary> + * Get the ith token from the current position 1..n where k=1 is the + * first symbol of lookahead. + * </summary> + */ + public SlimToken LT( int k ) + { + if ( p == -1 ) + { + FillBuffer(); + } + if ( k == 0 ) + { + return default( SlimToken ); + } + if ( k < 0 ) + { + return LB( -k ); + } + //System.out.print("LT(p="+p+","+k+")="); + if ( ( p + k - 1 ) >= tokens.Count ) + { + return new SlimToken(TokenTypes.EndOfFile); + } + //System.out.println(tokens.get(p+k-1)); + int i = p; + int n = 1; + // find k good tokens + while ( n < k ) + { + // skip off-channel tokens + i = SkipOffTokenChannels( i + 1 ); // leave p on valid token + n++; + } + if ( i >= tokens.Count ) + { + return new SlimToken(TokenTypes.EndOfFile); + } + return tokens[i]; + } + + /** <summary>Look backwards k tokens on-channel tokens</summary> */ + protected SlimToken LB( int k ) + { + //System.out.print("LB(p="+p+","+k+") "); + if ( p == -1 ) + { + FillBuffer(); + } + if ( k == 0 ) + { + return default( SlimToken ); + } + if ( ( p - k ) < 0 ) + { + return default( SlimToken ); + } + + int i = p; + int n = 1; + // find k good tokens looking backwards + while ( n <= k ) + { + // skip off-channel tokens + i = SkipOffTokenChannelsReverse( i - 1 ); // leave p on valid token + n++; + } + if ( i < 0 ) + { + return default( SlimToken ); + } + return tokens[i]; + } + + /** <summary> + * Return absolute token i; ignore which channel the tokens are on; + * that is, count all tokens not just on-channel tokens. + * </summary> + */ + public SlimToken Get( int i ) + { + return tokens[i]; + } + + public int LA( int i ) + { + return LT( i ).Type; + } + + public int Mark() + { + if ( p == -1 ) + { + FillBuffer(); + } + lastMarker = Index; + return lastMarker; + } + + public void Release( int marker ) + { + // no resources to release + } + + public int Count + { + get + { + return tokens.Count; + } + } + + public void Rewind( int marker ) + { + Seek( marker ); + } + + public void Rewind() + { + Seek( lastMarker ); + } + + public void Reset() + { + p = 0; + lastMarker = 0; + } + + public void Seek( int index ) + { + p = index; + } + + public ITokenSource<SlimToken> TokenSource + { + get + { + return _tokenSource; + } + } + + public string SourceName + { + get + { + return TokenSource.SourceName; + } + } + + public override string ToString() + { + if ( p == -1 ) + { + throw new InvalidOperationException( "Buffer is not yet filled." ); + } + return ToString( 0, tokens.Count - 1 ); + } + + public virtual string ToString( int start, int stop ) + { + if ( start < 0 || stop < 0 ) + { + return null; + } + if ( p == -1 ) + { + throw new InvalidOperationException( "Buffer is not yet filled." ); + } + if ( stop >= tokens.Count ) + { + stop = tokens.Count - 1; + } + StringBuilder buf = new StringBuilder(); + for ( int i = start; i <= stop; i++ ) + { + SlimToken t = tokens[i]; + SlimLexer lexer = _tokenSource as SlimLexer; + if ( lexer != null ) + { + SlimStringStream input = lexer.CharStream as SlimStringStream; + if ( input != null ) + { + string text = input.Substring( t.StartIndex, t.StopIndex - t.StartIndex + 1 ); + buf.Append( text ); + } + } + } + return buf.ToString(); + } + + public virtual string ToString( IToken start, IToken stop ) + { + if ( start != null && stop != null ) + { + return ToString( start.TokenIndex, stop.TokenIndex ); + } + return null; + } + } +} |