diff options
Diffstat (limited to 'runtime/CSharp3/Sources/Antlr3.Runtime/LegacyCommonTokenStream.cs')
| -rw-r--r-- | runtime/CSharp3/Sources/Antlr3.Runtime/LegacyCommonTokenStream.cs | 526 |
1 files changed, 0 insertions, 526 deletions
diff --git a/runtime/CSharp3/Sources/Antlr3.Runtime/LegacyCommonTokenStream.cs b/runtime/CSharp3/Sources/Antlr3.Runtime/LegacyCommonTokenStream.cs deleted file mode 100644 index 936f586..0000000 --- a/runtime/CSharp3/Sources/Antlr3.Runtime/LegacyCommonTokenStream.cs +++ /dev/null @@ -1,526 +0,0 @@ -/* - * [The "BSD licence"] - * Copyright (c) 2005-2008 Terence Parr - * All rights reserved. - * - * Conversion to C#: - * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -namespace Antlr.Runtime -{ - using System.Collections.Generic; - - using InvalidOperationException = System.InvalidOperationException; - using StringBuilder = System.Text.StringBuilder; - - /** <summary> - * The most common stream of tokens is one where every token is buffered up - * and tokens are prefiltered for a certain channel (the parser will only - * see these tokens and cannot change the filter channel number during the - * parse). - * </summary> - * - * <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks> - */ - [System.Serializable] - public class LegacyCommonTokenStream : ITokenStream - { - [System.NonSerialized] - ITokenSource _tokenSource; - - /** <summary> - * Record every single token pulled from the source so we can reproduce - * chunks of it later. - * </summary> - */ - protected List<IToken> tokens; - - /** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */ - protected IDictionary<int, int> channelOverrideMap; - - /** <summary>Set<tokentype>; discard any tokens with this type</summary> */ - protected List<int> discardSet; - - /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */ - protected int channel = TokenChannels.Default; - - /** <summary>By default, track all incoming tokens</summary> */ - protected bool discardOffChannelTokens = false; - - /** <summary>Track the last mark() call result value for use in rewind().</summary> */ - protected int lastMarker; - - /** <summary> - * The index into the tokens list of the current token (next token - * to consume). p==-1 indicates that the tokens list is empty - * </summary> - */ - protected int p = -1; - - public LegacyCommonTokenStream() - { - tokens = new List<IToken>( 500 ); - } - - public LegacyCommonTokenStream(ITokenSource tokenSource) - : this() - { - this._tokenSource = tokenSource; - } - - public LegacyCommonTokenStream( ITokenSource tokenSource, int channel ) - : this( tokenSource ) - { - this.channel = channel; - } - - public virtual int Index - { - get - { - return p; - } - } - - /// <summary> - /// How deep have we gone? - /// </summary> - public virtual int Range - { - get; - protected set; - } - - /** <summary>Reset this token stream by setting its token source.</summary> */ - public virtual void SetTokenSource( ITokenSource tokenSource ) - { - this._tokenSource = tokenSource; - tokens.Clear(); - p = -1; - channel = TokenChannels.Default; - } - - /** <summary> - * Load all tokens from the token source and put in tokens. - * This is done upon first LT request because you might want to - * set some token type / channel overrides before filling buffer. - * </summary> - */ - public virtual void FillBuffer() - { - // fast return if the buffer is already full - if ( p != -1 ) - return; - - int index = 0; - IToken t = _tokenSource.NextToken(); - while ( t != null && t.Type != CharStreamConstants.EndOfFile ) - { - bool discard = false; - // is there a channel override for token type? - int channelI; - if ( channelOverrideMap != null && channelOverrideMap.TryGetValue( t.Type, out channelI ) ) - t.Channel = channelI; - - //if ( channelOverrideMap != null && channelOverrideMap.ContainsKey( t.getType() ) ) - //{ - // object channelI = channelOverrideMap.get( t.getType() ); - // if ( channelI != null ) - // { - // t.setChannel( (int)channelI ); - // } - //} - if ( discardSet != null && - discardSet.Contains( t.Type ) ) - { - discard = true; - } - else if ( discardOffChannelTokens && t.Channel != this.channel ) - { - discard = true; - } - if ( !discard ) - { - t.TokenIndex = index; - tokens.Add( t ); - index++; - } - t = _tokenSource.NextToken(); - } - // leave p pointing at first token on channel - p = 0; - p = SkipOffTokenChannels( p ); - } - - /** <summary> - * Move the input pointer to the next incoming token. The stream - * must become active with LT(1) available. consume() simply - * moves the input pointer so that LT(1) points at the next - * input symbol. Consume at least one token. - * </summary> - * - * <remarks> - * Walk past any token not on the channel the parser is listening to. - * </remarks> - */ - public virtual void Consume() - { - if ( p < tokens.Count ) - { - p++; - p = SkipOffTokenChannels( p ); // leave p on valid token - } - } - - /** <summary>Given a starting index, return the index of the first on-channel token.</summary> */ - protected virtual int SkipOffTokenChannels( int i ) - { - int n = tokens.Count; - while ( i < n && ( (IToken)tokens[i] ).Channel != channel ) - { - i++; - } - return i; - } - - protected virtual int SkipOffTokenChannelsReverse( int i ) - { - while ( i >= 0 && ( (IToken)tokens[i] ).Channel != channel ) - { - i--; - } - return i; - } - - /** <summary> - * A simple filter mechanism whereby you can tell this token stream - * to force all tokens of type ttype to be on channel. For example, - * when interpreting, we cannot exec actions so we need to tell - * the stream to force all WS and NEWLINE to be a different, ignored - * channel. - * </summary> - */ - public virtual void SetTokenTypeChannel( int ttype, int channel ) - { - if ( channelOverrideMap == null ) - { - channelOverrideMap = new Dictionary<int, int>(); - } - channelOverrideMap[ttype] = channel; - } - - public virtual void DiscardTokenType( int ttype ) - { - if ( discardSet == null ) - { - discardSet = new List<int>(); - } - discardSet.Add( ttype ); - } - - public virtual void SetDiscardOffChannelTokens( bool discardOffChannelTokens ) - { - this.discardOffChannelTokens = discardOffChannelTokens; - } - - public virtual IList<IToken> GetTokens() - { - if ( p == -1 ) - { - FillBuffer(); - } - return tokens; - } - - public virtual IList<IToken> GetTokens( int start, int stop ) - { - return GetTokens( start, stop, (BitSet)null ); - } - - /** <summary> - * Given a start and stop index, return a List of all tokens in - * the token type BitSet. Return null if no tokens were found. This - * method looks at both on and off channel tokens. - * </summary> - */ - public virtual IList<IToken> GetTokens( int start, int stop, BitSet types ) - { - if ( p == -1 ) - { - FillBuffer(); - } - if ( stop >= tokens.Count ) - { - stop = tokens.Count - 1; - } - if ( start < 0 ) - { - start = 0; - } - if ( start > stop ) - { - return null; - } - - // list = tokens[start:stop]:{Token t, t.getType() in types} - IList<IToken> filteredTokens = new List<IToken>(); - for ( int i = start; i <= stop; i++ ) - { - IToken t = tokens[i]; - if ( types == null || types.Member( t.Type ) ) - { - filteredTokens.Add( t ); - } - } - if ( filteredTokens.Count == 0 ) - { - filteredTokens = null; - } - return filteredTokens; - } - - public virtual IList<IToken> GetTokens( int start, int stop, IList<int> types ) - { - return GetTokens( start, stop, new BitSet( types ) ); - } - - public virtual IList<IToken> GetTokens( int start, int stop, int ttype ) - { - return GetTokens( start, stop, BitSet.Of( ttype ) ); - } - - /** <summary> - * Get the ith token from the current position 1..n where k=1 is the - * first symbol of lookahead. - * </summary> - */ - public virtual IToken LT( int k ) - { - if ( p == -1 ) - { - FillBuffer(); - } - if ( k == 0 ) - { - return null; - } - if ( k < 0 ) - { - return LB( -k ); - } - //System.out.print("LT(p="+p+","+k+")="); - if ( ( p + k - 1 ) >= tokens.Count ) - { - return tokens[tokens.Count - 1]; - } - //System.out.println(tokens.get(p+k-1)); - int i = p; - int n = 1; - // find k good tokens - while ( n < k ) - { - // skip off-channel tokens - i = SkipOffTokenChannels( i + 1 ); // leave p on valid token - n++; - } - if ( i >= tokens.Count ) - { - return tokens[tokens.Count - 1]; - } - - if (i > Range) - Range = i; - - return (IToken)tokens[i]; - } - - /** <summary>Look backwards k tokens on-channel tokens</summary> */ - protected virtual IToken LB( int k ) - { - //System.out.print("LB(p="+p+","+k+") "); - if ( p == -1 ) - { - FillBuffer(); - } - if ( k == 0 ) - { - return null; - } - if ( ( p - k ) < 0 ) - { - return null; - } - - int i = p; - int n = 1; - // find k good tokens looking backwards - while ( n <= k ) - { - // skip off-channel tokens - i = SkipOffTokenChannelsReverse( i - 1 ); // leave p on valid token - n++; - } - if ( i < 0 ) - { - return null; - } - return (IToken)tokens[i]; - } - - /** <summary> - * Return absolute token i; ignore which channel the tokens are on; - * that is, count all tokens not just on-channel tokens. - * </summary> - */ - public virtual IToken Get( int i ) - { - return (IToken)tokens[i]; - } - -#if false - /** Get all tokens from start..stop inclusively */ - public virtual List<IToken> Get(int start, int count) - { - if (start < 0) - throw new ArgumentOutOfRangeException("start"); - if (count < 0) - throw new ArgumentOutOfRangeException("count"); - - if (p == -1) - FillBuffer(); - - return new List<IToken>(tokens.Skip(start).Take(count)); - } -#endif - - public virtual int LA( int i ) - { - return LT( i ).Type; - } - - public virtual int Mark() - { - if ( p == -1 ) - { - FillBuffer(); - } - lastMarker = Index; - return lastMarker; - } - - public virtual void Release( int marker ) - { - // no resources to release - } - - public virtual int Count - { - get - { - return tokens.Count; - } - } - - public virtual void Rewind( int marker ) - { - Seek( marker ); - } - - public virtual void Rewind() - { - Seek( lastMarker ); - } - - public virtual void Reset() - { - p = 0; - lastMarker = 0; - } - - public virtual void Seek( int index ) - { - p = index; - } - - public virtual ITokenSource TokenSource - { - get - { - return _tokenSource; - } - } - - public virtual string SourceName - { - get - { - return TokenSource.SourceName; - } - } - - public override string ToString() - { - if ( p == -1 ) - { - throw new InvalidOperationException( "Buffer is not yet filled." ); - } - return ToString( 0, tokens.Count - 1 ); - } - - public virtual string ToString( int start, int stop ) - { - if ( start < 0 || stop < 0 ) - { - return null; - } - if ( p == -1 ) - { - throw new InvalidOperationException( "Buffer is not yet filled." ); - } - if ( stop >= tokens.Count ) - { - stop = tokens.Count - 1; - } - StringBuilder buf = new StringBuilder(); - for ( int i = start; i <= stop; i++ ) - { - IToken t = tokens[i]; - buf.Append( t.Text ); - } - return buf.ToString(); - } - - public virtual string ToString( IToken start, IToken stop ) - { - if ( start != null && stop != null ) - { - return ToString( start.TokenIndex, stop.TokenIndex ); - } - return null; - } - } -} |