aboutsummaryrefslogtreecommitdiff
path: root/antlr-3.4/runtime/ActionScript/project/src/org/antlr/runtime/CommonTokenStream.as
diff options
context:
space:
mode:
Diffstat (limited to 'antlr-3.4/runtime/ActionScript/project/src/org/antlr/runtime/CommonTokenStream.as')
-rw-r--r--antlr-3.4/runtime/ActionScript/project/src/org/antlr/runtime/CommonTokenStream.as371
1 files changed, 371 insertions, 0 deletions
diff --git a/antlr-3.4/runtime/ActionScript/project/src/org/antlr/runtime/CommonTokenStream.as b/antlr-3.4/runtime/ActionScript/project/src/org/antlr/runtime/CommonTokenStream.as
new file mode 100644
index 0000000..afebc09
--- /dev/null
+++ b/antlr-3.4/runtime/ActionScript/project/src/org/antlr/runtime/CommonTokenStream.as
@@ -0,0 +1,371 @@
+/*
+ [The "BSD licence"]
+ Copyright (c) 2005-2006 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+package org.antlr.runtime {
+
+ /** The most common stream of tokens is one where every token is buffered up
+ * and tokens are prefiltered for a certain channel (the parser will only
+ * see these tokens and cannot change the filter channel number during the
+ * parse).
+ *
+ * TODO: how to access the full token stream? How to track all tokens matched per rule?
+ */
+ public class CommonTokenStream implements TokenStream {
+ protected var _tokenSource:TokenSource;
+
+ /** Record every single token pulled from the source so we can reproduce
+ * chunks of it later.
+ */
+ protected var tokens:Array = new Array();
+
+ /** Map<tokentype, channel> to override some Tokens' channel numbers */
+ protected var channelOverrideMap:Array;
+
+ /** Set<tokentype>; discard any tokens with this type */
+ protected var discardSet:Array;
+
+ /** Skip tokens on any channel but this one; this is how we skip whitespace... */
+ protected var channel:int = TokenConstants.DEFAULT_CHANNEL;
+
+ /** By default, track all incoming tokens */
+ protected var _discardOffChannelTokens:Boolean = false;
+
+ /** Track the last mark() call result value for use in rewind(). */
+ protected var lastMarker:int;
+
+ /** The index into the tokens list of the current token (next token
+ * to consume). p==-1 indicates that the tokens list is empty
+ */
+ protected var p:int = -1;
+
+ public function CommonTokenStream(tokenSource:TokenSource = null, channel:int = TokenConstants.DEFAULT_CHANNEL) {
+ _tokenSource = tokenSource;
+ this.channel = channel;
+ }
+
+ /** Reset this token stream by setting its token source. */
+ public function set tokenSource(tokenSource:TokenSource):void {
+ _tokenSource = tokenSource;
+ tokens.clear();
+ p = -1;
+ channel = TokenConstants.DEFAULT_CHANNEL;
+ }
+
+ /** Load all tokens from the token source and put in tokens.
+ * This is done upon first LT request because you might want to
+ * set some token type / channel overrides before filling buffer.
+ */
+ protected function fillBuffer():void {
+ var index:int = 0;
+ var t:Token = tokenSource.nextToken();
+ while ( t!=null && t.type != CharStreamConstants.EOF ) {
+ var discard:Boolean = false;
+ // is there a channel override for token type?
+ if ( channelOverrideMap != null ) {
+ if (channelOverrideMap[t.type] != undefined) {
+ t.channel = channelOverrideMap[t.type];
+ }
+ }
+ if ( discardSet !=null &&
+ discardSet[t.type] == true )
+ {
+ discard = true;
+ }
+ else if ( _discardOffChannelTokens && t.channel != this.channel ) {
+ discard = true;
+ }
+ if ( !discard ) {
+ t.tokenIndex = index;
+ tokens.push(t);
+ index++;
+ }
+ t = tokenSource.nextToken();
+ }
+ // leave p pointing at first token on channel
+ p = 0;
+ p = skipOffTokenChannels(p);
+ }
+
+ /** Move the input pointer to the next incoming token. The stream
+ * must become active with LT(1) available. consume() simply
+ * moves the input pointer so that LT(1) points at the next
+ * input symbol. Consume at least one token.
+ *
+ * Walk past any token not on the channel the parser is listening to.
+ */
+ public function consume():void {
+ if ( p<tokens.length ) {
+ p++;
+ p = skipOffTokenChannels(p); // leave p on valid token
+ }
+ }
+
+ /** Given a starting index, return the index of the first on-channel
+ * token.
+ */
+ protected function skipOffTokenChannels(i:int):int {
+ var n:int = tokens.length;
+ while ( i<n && (Token(tokens[i])).channel != channel) {
+ i++;
+ }
+ return i;
+ }
+
+ protected function skipOffTokenChannelsReverse(i:int):int {
+ while ( i>= 0 && (Token(tokens[i])).channel != channel) {
+ i--;
+ }
+ return i;
+ }
+
+ /** A simple filter mechanism whereby you can tell this token stream
+ * to force all tokens of type ttype to be on channel. For example,
+ * when interpreting, we cannot exec actions so we need to tell
+ * the stream to force all WS and NEWLINE to be a different, ignored
+ * channel.
+ */
+ public function setTokenTypeChannel(ttype:int, channel:int):void {
+ if ( channelOverrideMap==null ) {
+ channelOverrideMap = new Array();
+ }
+ channelOverrideMap[ttype] = channel;
+ }
+
+ public function discardTokenType(ttype:int):void {
+ if ( discardSet==null ) {
+ discardSet = new Array();
+ }
+ discardSet[ttype] = true;
+ }
+
+ public function discardOffChannelTokens(discardOffChannelTokens:Boolean):void {
+ _discardOffChannelTokens = discardOffChannelTokens;
+ }
+
+ public function getTokens():Array {
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ return tokens;
+ }
+
+ public function getTokensRange(start:int, stop:int):Array {
+ return getTokensBitSet(start, stop, null);
+ }
+
+ /** Given a start and stop index, return a List of all tokens in
+ * the token type BitSet. Return null if no tokens were found. This
+ * method looks at both on and off channel tokens.
+ *
+ * Renamed from getTokens
+ */
+ public function getTokensBitSet(start:int, stop:int, types:BitSet):Array {
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ if ( stop>=tokens.length ) {
+ stop=tokens.length-1;
+ }
+ if ( start<0 ) {
+ start=0;
+ }
+ if ( start>stop ) {
+ return null;
+ }
+
+ // list = tokens[start:stop]:{Token t, t.getType() in types}
+ var filteredTokens:Array = new Array();
+ for (var i:int=start; i<=stop; i++) {
+ var t:Token = Token(tokens[i]);
+ if ( types==null || types.member(t.type) ) {
+ filteredTokens.push(t);
+ }
+ }
+ if ( filteredTokens.length==0 ) {
+ filteredTokens = null;
+ }
+ return filteredTokens;
+ }
+
+ public function getTokensArray(start:int, stop:int, types:Array):Array {
+ return getTokensBitSet(start,stop,new BitSet(types));
+ }
+
+ public function getTokensInt(start:int, stop:int, ttype:int):Array {
+ return getTokensBitSet(start,stop,BitSet.of(ttype));
+ }
+
+ /** Get the ith token from the current position 1..n where k=1 is the
+ * first symbol of lookahead.
+ */
+ public function LT(k:int):Token {
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ if ( k==0 ) {
+ return null;
+ }
+ if ( k<0 ) {
+ return LB(-k);
+ }
+ //System.out.print("LT(p="+p+","+k+")=");
+ if ( (p+k-1) >= tokens.length ) {
+ return TokenConstants.EOF_TOKEN;
+ }
+ //System.out.println(tokens.get(p+k-1));
+ var i:int = p;
+ var n:int = 1;
+ // find k good tokens
+ while ( n<k ) {
+ // skip off-channel tokens
+ i = skipOffTokenChannels(i+1); // leave p on valid token
+ n++;
+ }
+ if ( i>=tokens.length ) {
+ return TokenConstants.EOF_TOKEN;
+ }
+ return Token(tokens[i]);
+ }
+
+ /** Look backwards k tokens on-channel tokens */
+ protected function LB(k:int):Token {
+ //System.out.print("LB(p="+p+","+k+") ");
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ if ( k==0 ) {
+ return null;
+ }
+ if ( (p-k)<0 ) {
+ return null;
+ }
+
+ var i:int = p;
+ var n:int = 1;
+ // find k good tokens looking backwards
+ while ( n<=k ) {
+ // skip off-channel tokens
+ i = skipOffTokenChannelsReverse(i-1); // leave p on valid token
+ n++;
+ }
+ if ( i<0 ) {
+ return null;
+ }
+ return Token(tokens[i]);
+ }
+
+ /** Return absolute token i; ignore which channel the tokens are on;
+ * that is, count all tokens not just on-channel tokens.
+ */
+ public function getToken(i:int):Token {
+ return Token(tokens[i]);
+ }
+
+ public function LA(i:int):int {
+ return LT(i).type;
+ }
+
+ public function mark():int {
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ lastMarker = index;
+ return lastMarker;
+ }
+
+ public function release(marker:int):void {
+ // no resources to release
+ }
+
+ public function get size():int {
+ return tokens.length;
+ }
+
+ public function get index():int {
+ return p;
+ }
+
+ public function reset():void {
+ p = 0;
+ lastMarker = 0;
+ }
+
+ public function rewindTo(marker:int):void {
+ seek(marker);
+ }
+
+ public function rewind():void {
+ seek(lastMarker);
+ }
+
+ public function seek(index:int):void {
+ p = index;
+ }
+
+ public function get tokenSource():TokenSource {
+ return _tokenSource;
+ }
+
+ public function get sourceName():String {
+ return tokenSource.sourceName;
+ }
+
+ public function toString():String {
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ return toStringWithRange(0, tokens.length-1);
+ }
+
+ public function toStringWithRange(start:int, stop:int):String {
+ if ( start<0 || stop<0 ) {
+ return null;
+ }
+ if ( p == -1 ) {
+ fillBuffer();
+ }
+ if ( stop>=tokens.length ) {
+ stop = tokens.length-1;
+ }
+ var buf:String = "";
+ for (var i:int = start; i <= stop; i++) {
+ var t:Token = Token(tokens[i]);
+ buf += t.text;
+ }
+ return buf.toString();
+ }
+
+ public function toStringWithTokenRange(start:Token, stop:Token):String {
+ if ( start!=null && stop!=null ) {
+ return toStringWithRange(start.tokenIndex, stop.tokenIndex);
+ }
+ return null;
+ }
+ }
+
+
+} \ No newline at end of file
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-04 20:48:01 +0000