Uploading antlr runtime for doclava use.

Change-Id: Iea8a6dd3a2c46dd3c13a81da6aa94e01bf98b854

87 files changed, 14235 insertions, 0 deletions

diff --git a/Android.mk b/Android.mk
new file mode 100644
index 0000000..efe3a09
--- /dev/null
+++ b/Android.mk
@@ -0,0 +1,32 @@
+# Copyright (C) 2011 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH := $(call my-dir)
+include $(CLEAR_VARS)
+
+LOCAL_MODULE := antlr
+#LOCAL_SDK_VERSION := 8
+LOCAL_SRC_FILES := $(call all-java-files-under, src)
+LOCAL_MODULE_TAGS := optional
+
+include $(BUILD_HOST_JAVA_LIBRARY)
+
+# Also build a host-side library
+# include $(CLEAR_VARS)
+# 
+# LOCAL_SRC_FILES := $(call all-java-files-under, src)
+# LOCAL_MODULE := antlrlib
+# 
+# include $(BUILD_HOST_JAVA_LIBRARY)
+
diff --git a/src/org/antlr/runtime/ANTLRFileStream.java b/src/org/antlr/runtime/ANTLRFileStream.java
new file mode 100755
index 0000000..27ef58b
--- /dev/null
+++ b/src/org/antlr/runtime/ANTLRFileStream.java
@@ -0,0 +1,78 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.io.*;
+
+/** This is a char buffer stream that is loaded from a file
+ *  all at once when you construct the object.  This looks very
+ *  much like an ANTLReader or ANTLRInputStream, but it's a special case
+ *  since we know the exact size of the object to load.  We can avoid lots
+ *  of data copying. 
+ */
+public class ANTLRFileStream extends ANTLRStringStream {
+	protected String fileName;
+
+	public ANTLRFileStream(String fileName) throws IOException {
+		this(fileName, null);
+	}
+
+	public ANTLRFileStream(String fileName, String encoding) throws IOException {
+		this.fileName = fileName;
+		load(fileName, encoding);
+	}
+
+	public void load(String fileName, String encoding)
+		throws IOException
+	{
+		if ( fileName==null ) {
+			return;
+		}
+		File f = new File(fileName);
+		int size = (int)f.length();
+		InputStreamReader isr;
+		FileInputStream fis = new FileInputStream(fileName);
+		if ( encoding!=null ) {
+			isr = new InputStreamReader(fis, encoding);
+		}
+		else {
+			isr = new InputStreamReader(fis);
+		}
+		try {
+			data = new char[size];
+			super.n = isr.read(data);
+		}
+		finally {
+			isr.close();
+		}
+	}
+
+	public String getSourceName() {
+		return fileName;
+	}
+}
diff --git a/src/org/antlr/runtime/ANTLRInputStream.java b/src/org/antlr/runtime/ANTLRInputStream.java
new file mode 100755
index 0000000..e544aae
--- /dev/null
+++ b/src/org/antlr/runtime/ANTLRInputStream.java
@@ -0,0 +1,70 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.io.*;
+
+/** A kind of ReaderStream that pulls from an InputStream.
+ *  Useful for reading from stdin and specifying file encodings etc...
+  */
+public class ANTLRInputStream extends ANTLRReaderStream {
+	public ANTLRInputStream() {
+	}
+
+	public ANTLRInputStream(InputStream input) throws IOException {
+		this(input, null);
+	}
+
+	public ANTLRInputStream(InputStream input, int size) throws IOException {
+		this(input, size, null);
+	}
+
+	public ANTLRInputStream(InputStream input, String encoding) throws IOException {
+		this(input, INITIAL_BUFFER_SIZE, encoding);
+	}
+
+	public ANTLRInputStream(InputStream input, int size, String encoding) throws IOException {
+		this(input, size, READ_BUFFER_SIZE, encoding);
+	}
+
+	public ANTLRInputStream(InputStream input,
+							int size,
+							int readBufferSize,
+							String encoding)
+		throws IOException
+	{
+		InputStreamReader isr;
+		if ( encoding!=null ) {
+			isr = new InputStreamReader(input, encoding);
+		}
+		else {
+			isr = new InputStreamReader(input);
+		}
+		load(isr, size, readBufferSize);
+	}
+}
diff --git a/src/org/antlr/runtime/ANTLRReaderStream.java b/src/org/antlr/runtime/ANTLRReaderStream.java
new file mode 100755
index 0000000..24d51ad
--- /dev/null
+++ b/src/org/antlr/runtime/ANTLRReaderStream.java
@@ -0,0 +1,95 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.io.*;
+
+/** Vacuum all input from a Reader and then treat it like a StringStream.
+ *  Manage the buffer manually to avoid unnecessary data copying.
+ *
+ *  If you need encoding, use ANTLRInputStream.
+ */
+public class ANTLRReaderStream extends ANTLRStringStream {
+	public static final int READ_BUFFER_SIZE = 1024;
+	public static final int INITIAL_BUFFER_SIZE = 1024;
+
+	public ANTLRReaderStream() {
+	}
+
+	public ANTLRReaderStream(Reader r) throws IOException {
+		this(r, INITIAL_BUFFER_SIZE, READ_BUFFER_SIZE);
+	}
+
+	public ANTLRReaderStream(Reader r, int size) throws IOException {
+		this(r, size, READ_BUFFER_SIZE);
+	}
+
+	public ANTLRReaderStream(Reader r, int size, int readChunkSize) throws IOException {
+		load(r, size, readChunkSize);
+	}
+
+	public void load(Reader r, int size, int readChunkSize)
+		throws IOException
+	{
+		if ( r==null ) {
+			return;
+		}
+		if ( size<=0 ) {
+			size = INITIAL_BUFFER_SIZE;
+		}
+		if ( readChunkSize<=0 ) {
+			readChunkSize = READ_BUFFER_SIZE;
+		}
+		// System.out.println("load "+size+" in chunks of "+readChunkSize);
+		try {
+			// alloc initial buffer size.
+			data = new char[size];
+			// read all the data in chunks of readChunkSize
+			int numRead=0;
+			int p = 0;
+			do {
+				if ( p+readChunkSize > data.length ) { // overflow?
+					// System.out.println("### overflow p="+p+", data.length="+data.length);
+					char[] newdata = new char[data.length*2]; // resize
+					System.arraycopy(data, 0, newdata, 0, data.length);
+					data = newdata;
+				}
+				numRead = r.read(data, p, readChunkSize);
+				// System.out.println("read "+numRead+" chars; p was "+p+" is now "+(p+numRead));
+				p += numRead;
+			} while (numRead!=-1); // while not EOF
+			// set the actual size of the data available;
+			// EOF subtracted one above in p+=numRead; add one back
+			super.n = p+1;
+			//System.out.println("n="+n);
+		}
+		finally {
+			r.close();
+		}
+	}
+}
diff --git a/src/org/antlr/runtime/ANTLRStringStream.java b/src/org/antlr/runtime/ANTLRStringStream.java
new file mode 100755
index 0000000..17af23c
--- /dev/null
+++ b/src/org/antlr/runtime/ANTLRStringStream.java
@@ -0,0 +1,230 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** A pretty quick CharStream that pulls all data from an array
+ *  directly.  Every method call counts in the lexer.  Java's
+ *  strings aren't very good so I'm avoiding.
+ */
+public class ANTLRStringStream implements CharStream {
+	/** The data being scanned */
+	protected char[] data;
+
+	/** How many characters are actually in the buffer */
+	protected int n;
+
+	/** 0..n-1 index into string of next char */
+	protected int p=0;
+
+	/** line number 1..n within the input */
+	protected int line = 1;
+
+	/** The index of the character relative to the beginning of the line 0..n-1 */
+	protected int charPositionInLine = 0;
+
+	/** tracks how deep mark() calls are nested */
+	protected int markDepth = 0;
+
+	/** A list of CharStreamState objects that tracks the stream state
+	 *  values line, charPositionInLine, and p that can change as you
+	 *  move through the input stream.  Indexed from 1..markDepth.
+     *  A null is kept @ index 0.  Create upon first call to mark().
+	 */
+	protected List markers;
+
+	/** Track the last mark() call result value for use in rewind(). */
+	protected int lastMarker;
+
+	/** What is name or source of this char stream? */
+	public String name;
+
+	public ANTLRStringStream() {
+	}
+
+	/** Copy data in string to a local char array */
+	public ANTLRStringStream(String input) {
+		this();
+		this.data = input.toCharArray();
+		this.n = input.length();
+	}
+
+	/** This is the preferred constructor as no data is copied */
+	public ANTLRStringStream(char[] data, int numberOfActualCharsInArray) {
+		this();
+		this.data = data;
+		this.n = numberOfActualCharsInArray;
+	}
+
+	/** Reset the stream so that it's in the same state it was
+	 *  when the object was created *except* the data array is not
+	 *  touched.
+	 */
+	public void reset() {
+		p = 0;
+		line = 1;
+		charPositionInLine = 0;
+		markDepth = 0;
+	}
+
+    public void consume() {
+		//System.out.println("prev p="+p+", c="+(char)data[p]);
+        if ( p < n ) {
+			charPositionInLine++;
+			if ( data[p]=='\n' ) {
+				/*
+				System.out.println("newline char found on line: "+line+
+								   "@ pos="+charPositionInLine);
+				*/
+				line++;
+				charPositionInLine=0;
+			}
+            p++;
+			//System.out.println("p moves to "+p+" (c='"+(char)data[p]+"')");
+        }
+    }
+
+    public int LA(int i) {
+		if ( i==0 ) {
+			return 0; // undefined
+		}
+		if ( i<0 ) {
+			i++; // e.g., translate LA(-1) to use offset i=0; then data[p+0-1]
+			if ( (p+i-1) < 0 ) {
+				return CharStream.EOF; // invalid; no char before first char
+			}
+		}
+
+		if ( (p+i-1) >= n ) {
+            //System.out.println("char LA("+i+")=EOF; p="+p);
+            return CharStream.EOF;
+        }
+        //System.out.println("char LA("+i+")="+(char)data[p+i-1]+"; p="+p);
+		//System.out.println("LA("+i+"); p="+p+" n="+n+" data.length="+data.length);
+		return data[p+i-1];
+    }
+
+	public int LT(int i) {
+		return LA(i);
+	}
+
+	/** Return the current input symbol index 0..n where n indicates the
+     *  last symbol has been read.  The index is the index of char to
+	 *  be returned from LA(1).
+     */
+    public int index() {
+        return p;
+    }
+
+	public int size() {
+		return n;
+	}
+
+	public int mark() {
+        if ( markers==null ) {
+            markers = new ArrayList();
+            markers.add(null); // depth 0 means no backtracking, leave blank
+        }
+        markDepth++;
+		CharStreamState state = null;
+		if ( markDepth>=markers.size() ) {
+			state = new CharStreamState();
+			markers.add(state);
+		}
+		else {
+			state = (CharStreamState)markers.get(markDepth);
+		}
+		state.p = p;
+		state.line = line;
+		state.charPositionInLine = charPositionInLine;
+		lastMarker = markDepth;
+		return markDepth;
+    }
+
+    public void rewind(int m) {
+		CharStreamState state = (CharStreamState)markers.get(m);
+		// restore stream state
+		seek(state.p);
+		line = state.line;
+		charPositionInLine = state.charPositionInLine;
+		release(m);
+	}
+
+	public void rewind() {
+		rewind(lastMarker);
+	}
+
+	public void release(int marker) {
+		// unwind any other markers made after m and release m
+		markDepth = marker;
+		// release this marker
+		markDepth--;
+	}
+
+	/** consume() ahead until p==index; can't just set p=index as we must
+	 *  update line and charPositionInLine.
+	 */
+	public void seek(int index) {
+		if ( index<=p ) {
+			p = index; // just jump; don't update stream state (line, ...)
+			return;
+		}
+		// seek forward, consume until p hits index
+		while ( p<index ) {
+			consume();
+		}
+	}
+
+	public String substring(int start, int stop) {
+		return new String(data,start,stop-start+1);
+	}
+
+	public int getLine() {
+		return line;
+	}
+
+	public int getCharPositionInLine() {
+		return charPositionInLine;
+	}
+
+	public void setLine(int line) {
+		this.line = line;
+	}
+
+	public void setCharPositionInLine(int pos) {
+		this.charPositionInLine = pos;
+	}
+
+	public String getSourceName() {
+		return name;
+	}
+
+    public String toString() { return new String(data); }
+}
diff --git a/src/org/antlr/runtime/BaseRecognizer.java b/src/org/antlr/runtime/BaseRecognizer.java
new file mode 100755
index 0000000..667664d
--- /dev/null
+++ b/src/org/antlr/runtime/BaseRecognizer.java
@@ -0,0 +1,886 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** A generic recognizer that can handle recognizers generated from
+ *  lexer, parser, and tree grammars.  This is all the parsing
+ *  support code essentially; most of it is error recovery stuff and
+ *  backtracking.
+ */
+public abstract class BaseRecognizer {
+	public static final int MEMO_RULE_FAILED = -2;
+	public static final int MEMO_RULE_UNKNOWN = -1;
+	public static final int INITIAL_FOLLOW_STACK_SIZE = 100;
+
+	// copies from Token object for convenience in actions
+	public static final int DEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL;
+	public static final int HIDDEN = Token.HIDDEN_CHANNEL;
+
+	public static final String NEXT_TOKEN_RULE_NAME = "nextToken";
+
+	/** State of a lexer, parser, or tree parser are collected into a state
+	 *  object so the state can be shared.  This sharing is needed to
+	 *  have one grammar import others and share same error variables
+	 *  and other state variables.  It's a kind of explicit multiple
+	 *  inheritance via delegation of methods and shared state.
+	 */
+	protected RecognizerSharedState state;
+
+	public BaseRecognizer() {
+		state = new RecognizerSharedState();
+	}
+
+	public BaseRecognizer(RecognizerSharedState state) {
+		if ( state==null ) {
+			state = new RecognizerSharedState();
+		}
+		this.state = state;
+	}
+
+	/** reset the parser's state; subclasses must rewinds the input stream */
+	public void reset() {
+		// wack everything related to error recovery
+		if ( state==null ) {
+			return; // no shared state work to do
+		}
+		state._fsp = -1;
+		state.errorRecovery = false;
+		state.lastErrorIndex = -1;
+		state.failed = false;
+		state.syntaxErrors = 0;
+		// wack everything related to backtracking and memoization
+		state.backtracking = 0;
+		for (int i = 0; state.ruleMemo!=null && i < state.ruleMemo.length; i++) { // wipe cache
+			state.ruleMemo[i] = null;
+		}
+	}
+
+
+	/** Match current input symbol against ttype.  Attempt
+	 *  single token insertion or deletion error recovery.  If
+	 *  that fails, throw MismatchedTokenException.
+	 *
+	 *  To turn off single token insertion or deletion error
+	 *  recovery, override recoverFromMismatchedToken() and have it
+     *  throw an exception. See TreeParser.recoverFromMismatchedToken().
+     *  This way any error in a rule will cause an exception and
+     *  immediate exit from rule.  Rule would recover by resynchronizing
+     *  to the set of symbols that can follow rule ref.
+	 */
+	public Object match(IntStream input, int ttype, BitSet follow)
+		throws RecognitionException
+	{
+		//System.out.println("match "+((TokenStream)input).LT(1));
+		Object matchedSymbol = getCurrentInputSymbol(input);
+		if ( input.LA(1)==ttype ) {
+			input.consume();
+			state.errorRecovery = false;
+			state.failed = false;
+			return matchedSymbol;
+		}
+		if ( state.backtracking>0 ) {
+			state.failed = true;
+			return matchedSymbol;
+		}
+		matchedSymbol = recoverFromMismatchedToken(input, ttype, follow);
+		return matchedSymbol;
+	}
+
+	/** Match the wildcard: in a symbol */
+	public void matchAny(IntStream input) {
+		state.errorRecovery = false;
+		state.failed = false;
+		input.consume();
+	}
+
+	public boolean mismatchIsUnwantedToken(IntStream input, int ttype) {
+		return input.LA(2)==ttype;
+	}
+
+	public boolean mismatchIsMissingToken(IntStream input, BitSet follow) {
+		if ( follow==null ) {
+			// we have no information about the follow; we can only consume
+			// a single token and hope for the best
+			return false;
+		}
+		// compute what can follow this grammar element reference
+		if ( follow.member(Token.EOR_TOKEN_TYPE) ) {
+			BitSet viableTokensFollowingThisRule = computeContextSensitiveRuleFOLLOW();
+			follow = follow.or(viableTokensFollowingThisRule);
+            if ( state._fsp>=0 ) { // remove EOR if we're not the start symbol
+                follow.remove(Token.EOR_TOKEN_TYPE);
+            }
+		}
+		// if current token is consistent with what could come after set
+		// then we know we're missing a token; error recovery is free to
+		// "insert" the missing token
+
+		//System.out.println("viable tokens="+follow.toString(getTokenNames()));
+		//System.out.println("LT(1)="+((TokenStream)input).LT(1));
+
+		// BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR
+		// in follow set to indicate that the fall of the start symbol is
+		// in the set (EOF can follow).
+		if ( follow.member(input.LA(1)) || follow.member(Token.EOR_TOKEN_TYPE) ) {
+			//System.out.println("LT(1)=="+((TokenStream)input).LT(1)+" is consistent with what follows; inserting...");
+			return true;
+		}
+		return false;
+	}
+
+	/** Report a recognition problem.
+	 *
+	 *  This method sets errorRecovery to indicate the parser is recovering
+	 *  not parsing.  Once in recovery mode, no errors are generated.
+	 *  To get out of recovery mode, the parser must successfully match
+	 *  a token (after a resync).  So it will go:
+	 *
+	 * 		1. error occurs
+	 * 		2. enter recovery mode, report error
+	 * 		3. consume until token found in resynch set
+	 * 		4. try to resume parsing
+	 * 		5. next match() will reset errorRecovery mode
+	 *
+	 *  If you override, make sure to update syntaxErrors if you care about that.
+	 */
+	public void reportError(RecognitionException e) {
+		// if we've already reported an error and have not matched a token
+		// yet successfully, don't report any errors.
+		if ( state.errorRecovery ) {
+			//System.err.print("[SPURIOUS] ");
+			return;
+		}
+		state.syntaxErrors++; // don't count spurious
+		state.errorRecovery = true;
+
+		displayRecognitionError(this.getTokenNames(), e);
+	}
+
+	public void displayRecognitionError(String[] tokenNames,
+										RecognitionException e)
+	{
+		String hdr = getErrorHeader(e);
+		String msg = getErrorMessage(e, tokenNames);
+		emitErrorMessage(hdr+" "+msg);
+	}
+
+	/** What error message should be generated for the various
+	 *  exception types?
+	 *
+	 *  Not very object-oriented code, but I like having all error message
+	 *  generation within one method rather than spread among all of the
+	 *  exception classes. This also makes it much easier for the exception
+	 *  handling because the exception classes do not have to have pointers back
+	 *  to this object to access utility routines and so on. Also, changing
+	 *  the message for an exception type would be difficult because you
+	 *  would have to subclassing exception, but then somehow get ANTLR
+	 *  to make those kinds of exception objects instead of the default.
+	 *  This looks weird, but trust me--it makes the most sense in terms
+	 *  of flexibility.
+	 *
+	 *  For grammar debugging, you will want to override this to add
+	 *  more information such as the stack frame with
+	 *  getRuleInvocationStack(e, this.getClass().getName()) and,
+	 *  for no viable alts, the decision description and state etc...
+	 *
+	 *  Override this to change the message generated for one or more
+	 *  exception types.
+	 */
+	public String getErrorMessage(RecognitionException e, String[] tokenNames) {
+		String msg = e.getMessage();
+		if ( e instanceof UnwantedTokenException ) {
+			UnwantedTokenException ute = (UnwantedTokenException)e;
+			String tokenName="<unknown>";
+			if ( ute.expecting== Token.EOF ) {
+				tokenName = "EOF";
+			}
+			else {
+				tokenName = tokenNames[ute.expecting];
+			}
+			msg = "extraneous input "+getTokenErrorDisplay(ute.getUnexpectedToken())+
+				" expecting "+tokenName;
+		}
+		else if ( e instanceof MissingTokenException ) {
+			MissingTokenException mte = (MissingTokenException)e;
+			String tokenName="<unknown>";
+			if ( mte.expecting== Token.EOF ) {
+				tokenName = "EOF";
+			}
+			else {
+				tokenName = tokenNames[mte.expecting];
+			}
+			msg = "missing "+tokenName+" at "+getTokenErrorDisplay(e.token);
+		}
+		else if ( e instanceof MismatchedTokenException ) {
+			MismatchedTokenException mte = (MismatchedTokenException)e;
+			String tokenName="<unknown>";
+			if ( mte.expecting== Token.EOF ) {
+				tokenName = "EOF";
+			}
+			else {
+				tokenName = tokenNames[mte.expecting];
+			}
+			msg = "mismatched input "+getTokenErrorDisplay(e.token)+
+				" expecting "+tokenName;
+		}
+		else if ( e instanceof MismatchedTreeNodeException ) {
+			MismatchedTreeNodeException mtne = (MismatchedTreeNodeException)e;
+			String tokenName="<unknown>";
+			if ( mtne.expecting==Token.EOF ) {
+				tokenName = "EOF";
+			}
+			else {
+				tokenName = tokenNames[mtne.expecting];
+			}
+			msg = "mismatched tree node: "+mtne.node+
+				" expecting "+tokenName;
+		}
+		else if ( e instanceof NoViableAltException ) {
+			//NoViableAltException nvae = (NoViableAltException)e;
+			// for development, can add "decision=<<"+nvae.grammarDecisionDescription+">>"
+			// and "(decision="+nvae.decisionNumber+") and
+			// "state "+nvae.stateNumber
+			msg = "no viable alternative at input "+getTokenErrorDisplay(e.token);
+		}
+		else if ( e instanceof EarlyExitException ) {
+			//EarlyExitException eee = (EarlyExitException)e;
+			// for development, can add "(decision="+eee.decisionNumber+")"
+			msg = "required (...)+ loop did not match anything at input "+
+				getTokenErrorDisplay(e.token);
+		}
+		else if ( e instanceof MismatchedSetException ) {
+			MismatchedSetException mse = (MismatchedSetException)e;
+			msg = "mismatched input "+getTokenErrorDisplay(e.token)+
+				" expecting set "+mse.expecting;
+		}
+		else if ( e instanceof MismatchedNotSetException ) {
+			MismatchedNotSetException mse = (MismatchedNotSetException)e;
+			msg = "mismatched input "+getTokenErrorDisplay(e.token)+
+				" expecting set "+mse.expecting;
+		}
+		else if ( e instanceof FailedPredicateException ) {
+			FailedPredicateException fpe = (FailedPredicateException)e;
+			msg = "rule "+fpe.ruleName+" failed predicate: {"+
+				fpe.predicateText+"}?";
+		}
+		return msg;
+	}
+
+	/** Get number of recognition errors (lexer, parser, tree parser).  Each
+	 *  recognizer tracks its own number.  So parser and lexer each have
+	 *  separate count.  Does not count the spurious errors found between
+	 *  an error and next valid token match
+	 *
+	 *  See also reportError()
+	 */
+	public int getNumberOfSyntaxErrors() {
+		return state.syntaxErrors;
+	}
+
+	/** What is the error header, normally line/character position information? */
+	public String getErrorHeader(RecognitionException e) {
+		if ( getSourceName()!=null )
+			return getSourceName()+" line "+e.line+":"+e.charPositionInLine;
+				
+		return "line "+e.line+":"+e.charPositionInLine;
+	}
+
+	/** How should a token be displayed in an error message? The default
+	 *  is to display just the text, but during development you might
+	 *  want to have a lot of information spit out.  Override in that case
+	 *  to use t.toString() (which, for CommonToken, dumps everything about
+	 *  the token). This is better than forcing you to override a method in
+	 *  your token objects because you don't have to go modify your lexer
+	 *  so that it creates a new Java type.
+	 */
+	public String getTokenErrorDisplay(Token t) {
+		String s = t.getText();
+		if ( s==null ) {
+			if ( t.getType()==Token.EOF ) {
+				s = "<EOF>";
+			}
+			else {
+				s = "<"+t.getType()+">";
+			}
+		}
+		s = s.replaceAll("\n","\\\\n");
+		s = s.replaceAll("\r","\\\\r");
+		s = s.replaceAll("\t","\\\\t");
+		return "'"+s+"'";
+	}
+
+	/** Override this method to change where error messages go */
+	public void emitErrorMessage(String msg) {
+		System.err.println(msg);
+	}
+
+	/** Recover from an error found on the input stream.  This is
+	 *  for NoViableAlt and mismatched symbol exceptions.  If you enable
+	 *  single token insertion and deletion, this will usually not
+	 *  handle mismatched symbol exceptions but there could be a mismatched
+	 *  token that the match() routine could not recover from.
+	 */
+	public void recover(IntStream input, RecognitionException re) {
+		if ( state.lastErrorIndex==input.index() ) {
+			// uh oh, another error at same token index; must be a case
+			// where LT(1) is in the recovery token set so nothing is
+			// consumed; consume a single token so at least to prevent
+			// an infinite loop; this is a failsafe.
+			input.consume();
+		}
+		state.lastErrorIndex = input.index();
+		BitSet followSet = computeErrorRecoverySet();
+		beginResync();
+		consumeUntil(input, followSet);
+		endResync();
+	}
+
+	/** A hook to listen in on the token consumption during error recovery.
+	 *  The DebugParser subclasses this to fire events to the listenter.
+	 */
+	public void beginResync() {
+	}
+
+	public void endResync() {
+	}
+
+	/*  Compute the error recovery set for the current rule.  During
+	 *  rule invocation, the parser pushes the set of tokens that can
+	 *  follow that rule reference on the stack; this amounts to
+	 *  computing FIRST of what follows the rule reference in the
+	 *  enclosing rule. This local follow set only includes tokens
+	 *  from within the rule; i.e., the FIRST computation done by
+	 *  ANTLR stops at the end of a rule.
+	 *
+	 *  EXAMPLE
+	 *
+	 *  When you find a "no viable alt exception", the input is not
+	 *  consistent with any of the alternatives for rule r.  The best
+	 *  thing to do is to consume tokens until you see something that
+	 *  can legally follow a call to r *or* any rule that called r.
+	 *  You don't want the exact set of viable next tokens because the
+	 *  input might just be missing a token--you might consume the
+	 *  rest of the input looking for one of the missing tokens.
+	 *
+	 *  Consider grammar:
+	 *
+	 *  a : '[' b ']'
+	 *    | '(' b ')'
+	 *    ;
+	 *  b : c '^' INT ;
+	 *  c : ID
+	 *    | INT
+	 *    ;
+	 *
+	 *  At each rule invocation, the set of tokens that could follow
+	 *  that rule is pushed on a stack.  Here are the various "local"
+	 *  follow sets:
+	 *
+	 *  FOLLOW(b1_in_a) = FIRST(']') = ']'
+	 *  FOLLOW(b2_in_a) = FIRST(')') = ')'
+	 *  FOLLOW(c_in_b) = FIRST('^') = '^'
+	 *
+	 *  Upon erroneous input "[]", the call chain is
+	 *
+	 *  a -> b -> c
+	 *
+	 *  and, hence, the follow context stack is:
+	 *
+	 *  depth  local follow set     after call to rule
+	 *    0         <EOF>                    a (from main())
+	 *    1          ']'                     b
+	 *    3          '^'                     c
+	 *
+	 *  Notice that ')' is not included, because b would have to have
+	 *  been called from a different context in rule a for ')' to be
+	 *  included.
+	 *
+	 *  For error recovery, we cannot consider FOLLOW(c)
+	 *  (context-sensitive or otherwise).  We need the combined set of
+	 *  all context-sensitive FOLLOW sets--the set of all tokens that
+	 *  could follow any reference in the call chain.  We need to
+	 *  resync to one of those tokens.  Note that FOLLOW(c)='^' and if
+	 *  we resync'd to that token, we'd consume until EOF.  We need to
+	 *  sync to context-sensitive FOLLOWs for a, b, and c: {']','^'}.
+	 *  In this case, for input "[]", LA(1) is in this set so we would
+	 *  not consume anything and after printing an error rule c would
+	 *  return normally.  It would not find the required '^' though.
+	 *  At this point, it gets a mismatched token error and throws an
+	 *  exception (since LA(1) is not in the viable following token
+	 *  set).  The rule exception handler tries to recover, but finds
+	 *  the same recovery set and doesn't consume anything.  Rule b
+	 *  exits normally returning to rule a.  Now it finds the ']' (and
+	 *  with the successful match exits errorRecovery mode).
+	 *
+	 *  So, you cna see that the parser walks up call chain looking
+	 *  for the token that was a member of the recovery set.
+	 *
+	 *  Errors are not generated in errorRecovery mode.
+	 *
+	 *  ANTLR's error recovery mechanism is based upon original ideas:
+	 *
+	 *  "Algorithms + Data Structures = Programs" by Niklaus Wirth
+	 *
+	 *  and
+	 *
+	 *  "A note on error recovery in recursive descent parsers":
+	 *  http://portal.acm.org/citation.cfm?id=947902.947905
+	 *
+	 *  Later, Josef Grosch had some good ideas:
+	 *
+	 *  "Efficient and Comfortable Error Recovery in Recursive Descent
+	 *  Parsers":
+	 *  ftp://www.cocolab.com/products/cocktail/doca4.ps/ell.ps.zip
+	 *
+	 *  Like Grosch I implemented local FOLLOW sets that are combined
+	 *  at run-time upon error to avoid overhead during parsing.
+	 */
+	protected BitSet computeErrorRecoverySet() {
+		return combineFollows(false);
+	}
+
+	/** Compute the context-sensitive FOLLOW set for current rule.
+	 *  This is set of token types that can follow a specific rule
+	 *  reference given a specific call chain.  You get the set of
+	 *  viable tokens that can possibly come next (lookahead depth 1)
+	 *  given the current call chain.  Contrast this with the
+	 *  definition of plain FOLLOW for rule r:
+	 *
+	 *   FOLLOW(r)={x | S=>*alpha r beta in G and x in FIRST(beta)}
+	 *
+	 *  where x in T* and alpha, beta in V*; T is set of terminals and
+	 *  V is the set of terminals and nonterminals.  In other words,
+	 *  FOLLOW(r) is the set of all tokens that can possibly follow
+	 *  references to r in *any* sentential form (context).  At
+	 *  runtime, however, we know precisely which context applies as
+	 *  we have the call chain.  We may compute the exact (rather
+	 *  than covering superset) set of following tokens.
+	 *
+	 *  For example, consider grammar:
+	 *
+	 *  stat : ID '=' expr ';'      // FOLLOW(stat)=={EOF}
+	 *       | "return" expr '.'
+	 *       ;
+	 *  expr : atom ('+' atom)* ;   // FOLLOW(expr)=={';','.',')'}
+	 *  atom : INT                  // FOLLOW(atom)=={'+',')',';','.'}
+	 *       | '(' expr ')'
+	 *       ;
+	 *
+	 *  The FOLLOW sets are all inclusive whereas context-sensitive
+	 *  FOLLOW sets are precisely what could follow a rule reference.
+	 *  For input input "i=(3);", here is the derivation:
+	 *
+	 *  stat => ID '=' expr ';'
+	 *       => ID '=' atom ('+' atom)* ';'
+	 *       => ID '=' '(' expr ')' ('+' atom)* ';'
+	 *       => ID '=' '(' atom ')' ('+' atom)* ';'
+	 *       => ID '=' '(' INT ')' ('+' atom)* ';'
+	 *       => ID '=' '(' INT ')' ';'
+	 *
+	 *  At the "3" token, you'd have a call chain of
+	 *
+	 *    stat -> expr -> atom -> expr -> atom
+	 *
+	 *  What can follow that specific nested ref to atom?  Exactly ')'
+	 *  as you can see by looking at the derivation of this specific
+	 *  input.  Contrast this with the FOLLOW(atom)={'+',')',';','.'}.
+	 *
+	 *  You want the exact viable token set when recovering from a
+	 *  token mismatch.  Upon token mismatch, if LA(1) is member of
+	 *  the viable next token set, then you know there is most likely
+	 *  a missing token in the input stream.  "Insert" one by just not
+	 *  throwing an exception.
+	 */
+	protected BitSet computeContextSensitiveRuleFOLLOW() {
+		return combineFollows(true);
+	}
+
+	// what is exact? it seems to only add sets from above on stack
+	// if EOR is in set i.  When it sees a set w/o EOR, it stops adding.
+	// Why would we ever want them all?  Maybe no viable alt instead of
+	// mismatched token?
+	protected BitSet combineFollows(boolean exact) {
+		int top = state._fsp;
+		BitSet followSet = new BitSet();
+		for (int i=top; i>=0; i--) {
+			BitSet localFollowSet = (BitSet)state.following[i];
+			/*
+			System.out.println("local follow depth "+i+"="+
+							   localFollowSet.toString(getTokenNames())+")");
+			 */
+			followSet.orInPlace(localFollowSet);
+			if ( exact ) {
+				// can we see end of rule?
+				if ( localFollowSet.member(Token.EOR_TOKEN_TYPE) ) {
+					// Only leave EOR in set if at top (start rule); this lets
+					// us know if have to include follow(start rule); i.e., EOF
+					if ( i>0 ) {
+						followSet.remove(Token.EOR_TOKEN_TYPE);
+					}
+				}
+				else { // can't see end of rule, quit
+					break;
+				}
+			}
+		}
+		return followSet;
+	}
+
+	/** Attempt to recover from a single missing or extra token.
+	 *
+	 *  EXTRA TOKEN
+	 *
+	 *  LA(1) is not what we are looking for.  If LA(2) has the right token,
+	 *  however, then assume LA(1) is some extra spurious token.  Delete it
+	 *  and LA(2) as if we were doing a normal match(), which advances the
+	 *  input.
+	 *
+	 *  MISSING TOKEN
+	 *
+	 *  If current token is consistent with what could come after
+	 *  ttype then it is ok to "insert" the missing token, else throw
+	 *  exception For example, Input "i=(3;" is clearly missing the
+	 *  ')'.  When the parser returns from the nested call to expr, it
+	 *  will have call chain:
+	 *
+	 *    stat -> expr -> atom
+	 *
+	 *  and it will be trying to match the ')' at this point in the
+	 *  derivation:
+	 *
+	 *       => ID '=' '(' INT ')' ('+' atom)* ';'
+	 *                          ^
+	 *  match() will see that ';' doesn't match ')' and report a
+	 *  mismatched token error.  To recover, it sees that LA(1)==';'
+	 *  is in the set of tokens that can follow the ')' token
+	 *  reference in rule atom.  It can assume that you forgot the ')'.
+	 */
+	protected Object recoverFromMismatchedToken(IntStream input, int ttype, BitSet follow)
+		throws RecognitionException
+	{
+		RecognitionException e = null;
+		// if next token is what we are looking for then "delete" this token
+		if ( mismatchIsUnwantedToken(input, ttype) ) {
+			e = new UnwantedTokenException(ttype, input);
+			/*
+			System.err.println("recoverFromMismatchedToken deleting "+
+							   ((TokenStream)input).LT(1)+
+							   " since "+((TokenStream)input).LT(2)+" is what we want");
+			 */
+			beginResync();
+			input.consume(); // simply delete extra token
+			endResync();
+			reportError(e);  // report after consuming so AW sees the token in the exception
+			// we want to return the token we're actually matching
+			Object matchedSymbol = getCurrentInputSymbol(input);
+			input.consume(); // move past ttype token as if all were ok
+			return matchedSymbol;
+		}
+		// can't recover with single token deletion, try insertion
+		if ( mismatchIsMissingToken(input, follow) ) {
+			Object inserted = getMissingSymbol(input, e, ttype, follow);
+			e = new MissingTokenException(ttype, input, inserted);
+			reportError(e);  // report after inserting so AW sees the token in the exception
+			return inserted;
+		}
+		// even that didn't work; must throw the exception
+		e = new MismatchedTokenException(ttype, input);
+		throw e;
+	}
+
+	/** Not currently used */
+	public Object recoverFromMismatchedSet(IntStream input,
+										   RecognitionException e,
+										   BitSet follow)
+		throws RecognitionException
+	{
+		if ( mismatchIsMissingToken(input, follow) ) {
+			// System.out.println("missing token");
+			reportError(e);
+			// we don't know how to conjure up a token for sets yet
+			return getMissingSymbol(input, e, Token.INVALID_TOKEN_TYPE, follow);
+		}
+		// TODO do single token deletion like above for Token mismatch
+		throw e;
+	}
+
+	/** Match needs to return the current input symbol, which gets put
+	 *  into the label for the associated token ref; e.g., x=ID.  Token
+	 *  and tree parsers need to return different objects. Rather than test
+	 *  for input stream type or change the IntStream interface, I use
+	 *  a simple method to ask the recognizer to tell me what the current
+	 *  input symbol is.
+	 * 
+	 *  This is ignored for lexers.
+	 */
+	protected Object getCurrentInputSymbol(IntStream input) { return null; }
+
+	/** Conjure up a missing token during error recovery.
+	 *
+	 *  The recognizer attempts to recover from single missing
+	 *  symbols. But, actions might refer to that missing symbol.
+	 *  For example, x=ID {f($x);}. The action clearly assumes
+	 *  that there has been an identifier matched previously and that
+	 *  $x points at that token. If that token is missing, but
+	 *  the next token in the stream is what we want we assume that
+	 *  this token is missing and we keep going. Because we
+	 *  have to return some token to replace the missing token,
+	 *  we have to conjure one up. This method gives the user control
+	 *  over the tokens returned for missing tokens. Mostly,
+	 *  you will want to create something special for identifier
+	 *  tokens. For literals such as '{' and ',', the default
+	 *  action in the parser or tree parser works. It simply creates
+	 *  a CommonToken of the appropriate type. The text will be the token.
+	 *  If you change what tokens must be created by the lexer,
+	 *  override this method to create the appropriate tokens.
+	 */
+	protected Object getMissingSymbol(IntStream input,
+									  RecognitionException e,
+									  int expectedTokenType,
+									  BitSet follow)
+	{
+		return null;
+	}
+
+	public void consumeUntil(IntStream input, int tokenType) {
+		//System.out.println("consumeUntil "+tokenType);
+		int ttype = input.LA(1);
+		while (ttype != Token.EOF && ttype != tokenType) {
+			input.consume();
+			ttype = input.LA(1);
+		}
+	}
+
+	/** Consume tokens until one matches the given token set */
+	public void consumeUntil(IntStream input, BitSet set) {
+		//System.out.println("consumeUntil("+set.toString(getTokenNames())+")");
+		int ttype = input.LA(1);
+		while (ttype != Token.EOF && !set.member(ttype) ) {
+			//System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]);
+			input.consume();
+			ttype = input.LA(1);
+		}
+	}
+
+	/** Push a rule's follow set using our own hardcoded stack */
+	protected void pushFollow(BitSet fset) {
+		if ( (state._fsp +1)>=state.following.length ) {
+			BitSet[] f = new BitSet[state.following.length*2];
+			System.arraycopy(state.following, 0, f, 0, state.following.length);
+			state.following = f;
+		}
+		state.following[++state._fsp] = fset;
+	}
+
+	/** Return List<String> of the rules in your parser instance
+	 *  leading up to a call to this method.  You could override if
+	 *  you want more details such as the file/line info of where
+	 *  in the parser java code a rule is invoked.
+	 *
+	 *  This is very useful for error messages and for context-sensitive
+	 *  error recovery.
+	 */
+	public List getRuleInvocationStack() {
+		String parserClassName = getClass().getName();
+		return getRuleInvocationStack(new Throwable(), parserClassName);
+	}
+
+	/** A more general version of getRuleInvocationStack where you can
+	 *  pass in, for example, a RecognitionException to get it's rule
+	 *  stack trace.  This routine is shared with all recognizers, hence,
+	 *  static.
+	 *
+	 *  TODO: move to a utility class or something; weird having lexer call this
+	 */
+	public static List getRuleInvocationStack(Throwable e,
+											  String recognizerClassName)
+	{
+		List rules = new ArrayList();
+		StackTraceElement[] stack = e.getStackTrace();
+		int i = 0;
+		for (i=stack.length-1; i>=0; i--) {
+			StackTraceElement t = stack[i];
+			if ( t.getClassName().startsWith("org.antlr.runtime.") ) {
+				continue; // skip support code such as this method
+			}
+			if ( t.getMethodName().equals(NEXT_TOKEN_RULE_NAME) ) {
+				continue;
+			}
+			if ( !t.getClassName().equals(recognizerClassName) ) {
+				continue; // must not be part of this parser
+			}
+            rules.add(t.getMethodName());
+		}
+		return rules;
+	}
+
+    public int getBacktrackingLevel() { return state.backtracking; }
+
+    public void setBacktrackingLevel(int n) { state.backtracking = n; }
+
+    /** Return whether or not a backtracking attempt failed. */
+    public boolean failed() { return state.failed; }
+
+	/** Used to print out token names like ID during debugging and
+	 *  error reporting.  The generated parsers implement a method
+	 *  that overrides this to point to their String[] tokenNames.
+	 */
+	public String[] getTokenNames() {
+		return null;
+	}
+
+	/** For debugging and other purposes, might want the grammar name.
+	 *  Have ANTLR generate an implementation for this method.
+	 */
+	public String getGrammarFileName() {
+		return null;
+	}
+
+	public abstract String getSourceName();
+
+	/** A convenience method for use most often with template rewrites.
+	 *  Convert a List<Token> to List<String>
+	 */
+	public List toStrings(List tokens) {
+		if ( tokens==null ) return null;
+		List strings = new ArrayList(tokens.size());
+		for (int i=0; i<tokens.size(); i++) {
+			strings.add(((Token)tokens.get(i)).getText());
+		}
+		return strings;
+	}
+
+	/** Given a rule number and a start token index number, return
+	 *  MEMO_RULE_UNKNOWN if the rule has not parsed input starting from
+	 *  start index.  If this rule has parsed input starting from the
+	 *  start index before, then return where the rule stopped parsing.
+	 *  It returns the index of the last token matched by the rule.
+	 *
+	 *  For now we use a hashtable and just the slow Object-based one.
+	 *  Later, we can make a special one for ints and also one that
+	 *  tosses out data after we commit past input position i.
+	 */
+	public int getRuleMemoization(int ruleIndex, int ruleStartIndex) {
+		if ( state.ruleMemo[ruleIndex]==null ) {
+			state.ruleMemo[ruleIndex] = new HashMap();
+		}
+		Integer stopIndexI =
+			(Integer)state.ruleMemo[ruleIndex].get(new Integer(ruleStartIndex));
+		if ( stopIndexI==null ) {
+			return MEMO_RULE_UNKNOWN;
+		}
+		return stopIndexI.intValue();
+	}
+
+	/** Has this rule already parsed input at the current index in the
+	 *  input stream?  Return the stop token index or MEMO_RULE_UNKNOWN.
+	 *  If we attempted but failed to parse properly before, return
+	 *  MEMO_RULE_FAILED.
+	 *
+	 *  This method has a side-effect: if we have seen this input for
+	 *  this rule and successfully parsed before, then seek ahead to
+	 *  1 past the stop token matched for this rule last time.
+	 */
+	public boolean alreadyParsedRule(IntStream input, int ruleIndex) {
+		int stopIndex = getRuleMemoization(ruleIndex, input.index());
+		if ( stopIndex==MEMO_RULE_UNKNOWN ) {
+			return false;
+		}
+		if ( stopIndex==MEMO_RULE_FAILED ) {
+			//System.out.println("rule "+ruleIndex+" will never succeed");
+			state.failed=true;
+		}
+		else {
+			//System.out.println("seen rule "+ruleIndex+" before; skipping ahead to @"+(stopIndex+1)+" failed="+state.failed);
+			input.seek(stopIndex+1); // jump to one past stop token
+		}
+		return true;
+	}
+
+	/** Record whether or not this rule parsed the input at this position
+	 *  successfully.  Use a standard java hashtable for now.
+	 */
+	public void memoize(IntStream input,
+						int ruleIndex,
+						int ruleStartIndex)
+	{
+		int stopTokenIndex = state.failed?MEMO_RULE_FAILED:input.index()-1;
+		if ( state.ruleMemo==null ) {
+			System.err.println("!!!!!!!!! memo array is null for "+ getGrammarFileName());
+		}
+		if ( ruleIndex >= state.ruleMemo.length ) {
+			System.err.println("!!!!!!!!! memo size is "+state.ruleMemo.length+", but rule index is "+ruleIndex);
+		}
+		if ( state.ruleMemo[ruleIndex]!=null ) {
+			state.ruleMemo[ruleIndex].put(
+				new Integer(ruleStartIndex), new Integer(stopTokenIndex)
+			);
+		}
+	}
+
+	/** return how many rule/input-index pairs there are in total.
+	 *  TODO: this includes synpreds. :(
+	 */
+	public int getRuleMemoizationCacheSize() {
+		int n = 0;
+		for (int i = 0; state.ruleMemo!=null && i < state.ruleMemo.length; i++) {
+			Map ruleMap = state.ruleMemo[i];
+			if ( ruleMap!=null ) {
+				n += ruleMap.size(); // how many input indexes are recorded?
+			}
+		}
+		return n;
+	}
+
+	public void traceIn(String ruleName, int ruleIndex, Object inputSymbol)  {
+		System.out.print("enter "+ruleName+" "+inputSymbol);
+		if ( state.backtracking>0 ) {
+			System.out.print(" backtracking="+state.backtracking);
+		}
+		System.out.println();
+	}
+
+	public void traceOut(String ruleName,
+						 int ruleIndex,
+						 Object inputSymbol)
+	{
+		System.out.print("exit "+ruleName+" "+inputSymbol);
+		if ( state.backtracking>0 ) {
+            System.out.print(" backtracking="+state.backtracking);
+            if ( state.failed ) System.out.print(" failed");
+            else System.out.print(" succeeded");
+        }
+		System.out.println();
+	}
+
+}
diff --git a/src/org/antlr/runtime/BitSet.java b/src/org/antlr/runtime/BitSet.java
new file mode 100755
index 0000000..f013a26
--- /dev/null
+++ b/src/org/antlr/runtime/BitSet.java
@@ -0,0 +1,325 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.List;
+
+/**A stripped-down version of org.antlr.misc.BitSet that is just
+ * good enough to handle runtime requirements such as FOLLOW sets
+ * for automatic error recovery.
+ */
+public class BitSet implements Cloneable {
+    protected final static int BITS = 64;    // number of bits / long
+    protected final static int LOG_BITS = 6; // 2^6 == 64
+
+    /* We will often need to do a mod operator (i mod nbits).  Its
+     * turns out that, for powers of two, this mod operation is
+     * same as (i & (nbits-1)).  Since mod is slow, we use a
+     * precomputed mod mask to do the mod instead.
+     */
+    protected final static int MOD_MASK = BITS - 1;
+
+    /** The actual data bits */
+    protected long bits[];
+
+    /** Construct a bitset of size one word (64 bits) */
+    public BitSet() {
+        this(BITS);
+    }
+
+    /** Construction from a static array of longs */
+    public BitSet(long[] bits_) {
+        bits = bits_;
+    }
+
+	/** Construction from a list of integers */
+	public BitSet(List items) {
+		this();
+		for (int i = 0; i < items.size(); i++) {
+			Integer v = (Integer) items.get(i);
+			add(v.intValue());
+		}
+	}
+
+    /** Construct a bitset given the size
+     * @param nbits The size of the bitset in bits
+     */
+    public BitSet(int nbits) {
+        bits = new long[((nbits - 1) >> LOG_BITS) + 1];
+    }
+
+	public static BitSet of(int el) {
+		BitSet s = new BitSet(el + 1);
+		s.add(el);
+		return s;
+	}
+
+	public static BitSet of(int a, int b) {
+		BitSet s = new BitSet(Math.max(a,b)+1);
+		s.add(a);
+		s.add(b);
+		return s;
+	}
+
+	public static BitSet of(int a, int b, int c) {
+		BitSet s = new BitSet();
+		s.add(a);
+		s.add(b);
+		s.add(c);
+		return s;
+	}
+
+	public static BitSet of(int a, int b, int c, int d) {
+		BitSet s = new BitSet();
+		s.add(a);
+		s.add(b);
+		s.add(c);
+		s.add(d);
+		return s;
+	}
+
+	/** return this | a in a new set */
+	public BitSet or(BitSet a) {
+		if ( a==null ) {
+			return this;
+		}
+		BitSet s = (BitSet)this.clone();
+		s.orInPlace(a);
+		return s;
+	}
+
+	/** or this element into this set (grow as necessary to accommodate) */
+	public void add(int el) {
+		int n = wordNumber(el);
+		if (n >= bits.length) {
+			growToInclude(el);
+		}
+		bits[n] |= bitMask(el);
+	}
+
+	/**
+	 * Grows the set to a larger number of bits.
+	 * @param bit element that must fit in set
+	 */
+	public void growToInclude(int bit) {
+		int newSize = Math.max(bits.length << 1, numWordsToHold(bit));
+		long newbits[] = new long[newSize];
+		System.arraycopy(bits, 0, newbits, 0, bits.length);
+		bits = newbits;
+	}
+
+	public void orInPlace(BitSet a) {
+		if ( a==null ) {
+			return;
+		}
+		// If this is smaller than a, grow this first
+		if (a.bits.length > bits.length) {
+			setSize(a.bits.length);
+		}
+		int min = Math.min(bits.length, a.bits.length);
+		for (int i = min - 1; i >= 0; i--) {
+			bits[i] |= a.bits[i];
+		}
+	}
+
+	/**
+	 * Sets the size of a set.
+	 * @param nwords how many words the new set should be
+	 */
+	private void setSize(int nwords) {
+		long newbits[] = new long[nwords];
+		int n = Math.min(nwords, bits.length);
+		System.arraycopy(bits, 0, newbits, 0, n);
+		bits = newbits;
+	}
+
+    private final static long bitMask(int bitNumber) {
+        int bitPosition = bitNumber & MOD_MASK; // bitNumber mod BITS
+        return 1L << bitPosition;
+    }
+
+    public Object clone() {
+        BitSet s;
+        try {
+            s = (BitSet)super.clone();
+            s.bits = new long[bits.length];
+            System.arraycopy(bits, 0, s.bits, 0, bits.length);
+        }
+        catch (CloneNotSupportedException e) {
+            throw new InternalError();
+        }
+        return s;
+    }
+
+    public int size() {
+        int deg = 0;
+        for (int i = bits.length - 1; i >= 0; i--) {
+            long word = bits[i];
+            if (word != 0L) {
+                for (int bit = BITS - 1; bit >= 0; bit--) {
+                    if ((word & (1L << bit)) != 0) {
+                        deg++;
+                    }
+                }
+            }
+        }
+        return deg;
+    }
+
+    public boolean equals(Object other) {
+        if ( other == null || !(other instanceof BitSet) ) {
+            return false;
+        }
+
+        BitSet otherSet = (BitSet)other;
+
+        int n = Math.min(this.bits.length, otherSet.bits.length);
+
+        // for any bits in common, compare
+        for (int i=0; i<n; i++) {
+            if (this.bits[i] != otherSet.bits[i]) {
+                return false;
+            }
+        }
+
+        // make sure any extra bits are off
+
+        if (this.bits.length > n) {
+            for (int i = n+1; i<this.bits.length; i++) {
+                if (this.bits[i] != 0) {
+                    return false;
+                }
+            }
+        }
+        else if (otherSet.bits.length > n) {
+            for (int i = n+1; i<otherSet.bits.length; i++) {
+                if (otherSet.bits[i] != 0) {
+                    return false;
+                }
+            }
+        }
+
+        return true;
+    }
+
+    public boolean member(int el) {
+		if ( el<0 ) {
+			return false;
+		}
+        int n = wordNumber(el);
+        if (n >= bits.length) return false;
+        return (bits[n] & bitMask(el)) != 0;
+    }
+
+	// remove this element from this set
+	public void remove(int el) {
+		int n = wordNumber(el);
+		if (n < bits.length) {
+			bits[n] &= ~bitMask(el);
+		}
+	}
+
+    public boolean isNil() {
+        for (int i = bits.length - 1; i >= 0; i--) {
+            if (bits[i] != 0) return false;
+        }
+        return true;
+    }
+
+    private final int numWordsToHold(int el) {
+        return (el >> LOG_BITS) + 1;
+    }
+
+    public int numBits() {
+        return bits.length << LOG_BITS; // num words * bits per word
+    }
+
+    /** return how much space is being used by the bits array not
+     *  how many actually have member bits on.
+     */
+    public int lengthInLongWords() {
+        return bits.length;
+    }
+
+    /**Is this contained within a? */
+    /*
+	public boolean subset(BitSet a) {
+        if (a == null || !(a instanceof BitSet)) return false;
+        return this.and(a).equals(this);
+    }
+	*/
+
+    public int[] toArray() {
+        int[] elems = new int[size()];
+        int en = 0;
+        for (int i = 0; i < (bits.length << LOG_BITS); i++) {
+            if (member(i)) {
+                elems[en++] = i;
+            }
+        }
+        return elems;
+    }
+
+    public long[] toPackedArray() {
+        return bits;
+    }
+
+	private final static int wordNumber(int bit) {
+		return bit >> LOG_BITS; // bit / BITS
+	}
+
+	public String toString() {
+		return toString(null);
+	}
+
+	public String toString(String[] tokenNames) {
+		StringBuffer buf = new StringBuffer();
+		String separator = ",";
+		boolean havePrintedAnElement = false;
+		buf.append('{');
+
+		for (int i = 0; i < (bits.length << LOG_BITS); i++) {
+			if (member(i)) {
+				if (i > 0 && havePrintedAnElement ) {
+					buf.append(separator);
+				}
+				if ( tokenNames!=null ) {
+					buf.append(tokenNames[i]);
+				}
+				else {
+					buf.append(i);
+				}
+				havePrintedAnElement = true;
+			}
+		}
+		buf.append('}');
+		return buf.toString();
+	}
+
+
+}
diff --git a/src/org/antlr/runtime/BufferedTokenStream.java b/src/org/antlr/runtime/BufferedTokenStream.java
new file mode 100755
index 0000000..7742d4f
--- /dev/null
+++ b/src/org/antlr/runtime/BufferedTokenStream.java
@@ -0,0 +1,272 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.NoSuchElementException;
+
+/** Buffer all input tokens but do on-demand fetching of new tokens from
+ *  lexer. Useful when the parser or lexer has to set context/mode info before
+ *  proper lexing of future tokens. The ST template parser needs this,
+ *  for example, because it has to constantly flip back and forth between
+ *  inside/output templates. E.g., <names:{hi, <it>}> has to parse names
+ *  as part of an expression but "hi, <it>" as a nested template.
+ *
+ *  You can't use this stream if you pass whitespace or other off-channel
+ *  tokens to the parser. The stream can't ignore off-channel tokens.
+ *  (UnbufferedTokenStream is the same way.)
+ *
+ *  This is not a subclass of UnbufferedTokenStream because I don't want
+ *  to confuse small moving window of tokens it uses for the full buffer.
+ */
+public class BufferedTokenStream implements TokenStream {
+    protected TokenSource tokenSource;
+
+    /** Record every single token pulled from the source so we can reproduce
+     *  chunks of it later.  The buffer in LookaheadStream overlaps sometimes
+     *  as its moving window moves through the input.  This list captures
+     *  everything so we can access complete input text.
+     */
+    protected List<Token> tokens = new ArrayList<Token>(100);
+
+    /** Track the last mark() call result value for use in rewind(). */
+    protected int lastMarker;
+
+    /** The index into the tokens list of the current token (next token
+     *  to consume).  tokens[p] should be LT(1).  p=-1 indicates need
+     *  to initialize with first token.  The ctor doesn't get a token.
+     *  First call to LT(1) or whatever gets the first token and sets p=0;
+     */
+    protected int p = -1;
+
+	protected int range = -1; // how deep have we gone?
+
+    public BufferedTokenStream() {;}
+
+    public BufferedTokenStream(TokenSource tokenSource) {
+        this.tokenSource = tokenSource;
+    }
+
+    public TokenSource getTokenSource() { return tokenSource; }
+
+	public int index() { return p; }
+
+	public int range() { return range; }
+
+    public int mark() {
+        if ( p == -1 ) setup();
+		lastMarker = index();
+		return lastMarker;
+	}
+
+	public void release(int marker) {
+		// no resources to release
+	}
+
+    public void rewind(int marker) {
+        seek(marker);
+    }
+
+    public void rewind() {
+        seek(lastMarker);
+    }
+
+    public void reset() {
+        p = 0;
+        lastMarker = 0;
+    }
+
+    public void seek(int index) { p = index; }
+
+    public int size() { return tokens.size(); }
+
+    /** Move the input pointer to the next incoming token.  The stream
+     *  must become active with LT(1) available.  consume() simply
+     *  moves the input pointer so that LT(1) points at the next
+     *  input symbol. Consume at least one token.
+     *
+     *  Walk past any token not on the channel the parser is listening to.
+     */
+    public void consume() {
+        if ( p == -1 ) setup();
+        p++;
+        sync(p);
+    }
+
+    /** Make sure index i in tokens has a token. */
+    protected void sync(int i) {
+        int n = i - tokens.size() + 1; // how many more elements we need?
+        //System.out.println("sync("+i+") needs "+n);
+        if ( n > 0 ) fetch(n);
+    }
+
+    /** add n elements to buffer */
+    protected void fetch(int n) {
+        for (int i=1; i<=n; i++) {
+            Token t = tokenSource.nextToken();
+            t.setTokenIndex(tokens.size());
+            //System.out.println("adding "+t+" at index "+tokens.size());
+            tokens.add(t);
+            if ( t.getType()==Token.EOF ) break;
+        }
+    }
+
+    public Token get(int i) {
+        if ( i < 0 || i >= tokens.size() ) {
+            throw new NoSuchElementException("token index "+i+" out of range 0.."+(tokens.size()-1));
+        }
+        return tokens.get(i);
+    }
+
+	/** Get all tokens from start..stop inclusively */
+	public List get(int start, int stop) {
+		if ( start<0 || stop<0 ) return null;
+		if ( p == -1 ) setup();
+		List subset = new ArrayList();
+		if ( stop>=tokens.size() ) stop = tokens.size()-1;
+		for (int i = start; i <= stop; i++) {
+			Token t = tokens.get(i);
+			if ( t.getType()==Token.EOF ) break;
+			subset.add(t);
+		}
+		return subset;
+	}
+
+	public int LA(int i) { return LT(i).getType(); }
+
+    protected Token LB(int k) {
+        if ( (p-k)<0 ) return null;
+        return tokens.get(p-k);
+    }
+
+    public Token LT(int k) {
+        if ( p == -1 ) setup();
+        if ( k==0 ) return null;
+        if ( k < 0 ) return LB(-k);
+
+		int i = p + k - 1;
+		sync(i);
+        if ( i >= tokens.size() ) { // return EOF token
+            // EOF must be last token
+            return tokens.get(tokens.size()-1);
+        }
+		if ( i>range ) range = i; 		
+        return tokens.get(i);
+    }
+
+    protected void setup() { sync(0); p = 0; }
+
+    /** Reset this token stream by setting its token source. */
+    public void setTokenSource(TokenSource tokenSource) {
+        this.tokenSource = tokenSource;
+        tokens.clear();
+        p = -1;
+    }
+    
+    public List getTokens() { return tokens; }
+
+    public List getTokens(int start, int stop) {
+        return getTokens(start, stop, (BitSet)null);
+    }
+
+    /** Given a start and stop index, return a List of all tokens in
+     *  the token type BitSet.  Return null if no tokens were found.  This
+     *  method looks at both on and off channel tokens.
+     */
+    public List getTokens(int start, int stop, BitSet types) {
+        if ( p == -1 ) setup();
+        if ( stop>=tokens.size() ) stop=tokens.size()-1;
+        if ( start<0 ) start=0;
+        if ( start>stop ) return null;
+
+        // list = tokens[start:stop]:{Token t, t.getType() in types}
+        List<Token> filteredTokens = new ArrayList<Token>();
+        for (int i=start; i<=stop; i++) {
+            Token t = tokens.get(i);
+            if ( types==null || types.member(t.getType()) ) {
+                filteredTokens.add(t);
+            }
+        }
+        if ( filteredTokens.size()==0 ) {
+            filteredTokens = null;
+        }
+        return filteredTokens;
+    }
+
+    public List getTokens(int start, int stop, List types) {
+        return getTokens(start,stop,new BitSet(types));
+    }
+
+    public List getTokens(int start, int stop, int ttype) {
+        return getTokens(start,stop,BitSet.of(ttype));
+    }
+
+    public String getSourceName() {	return tokenSource.getSourceName();	}
+
+    /** Grab *all* tokens from stream and return string */
+    public String toString() {
+        if ( p == -1 ) setup();
+        fill();
+        return toString(0, tokens.size()-1);
+    }
+
+    public String toString(int start, int stop) {
+        if ( start<0 || stop<0 ) return null;
+        if ( p == -1 ) setup();
+        if ( stop>=tokens.size() ) stop = tokens.size()-1;
+        StringBuffer buf = new StringBuffer();
+        for (int i = start; i <= stop; i++) {
+            Token t = tokens.get(i);
+            if ( t.getType()==Token.EOF ) break;
+            buf.append(t.getText());
+        }
+        return buf.toString();
+    }
+
+    public String toString(Token start, Token stop) {
+        if ( start!=null && stop!=null ) {
+            return toString(start.getTokenIndex(), stop.getTokenIndex());
+        }
+        return null;
+    }
+
+    /** Get all tokens from lexer until EOF */
+    public void fill() {
+        if ( p == -1 ) setup();
+        if ( tokens.get(p).getType()==Token.EOF ) return;
+
+        int i = p+1;
+        sync(i);
+        while ( tokens.get(i).getType()!=Token.EOF ) {
+            i++;
+            sync(i);
+        }
+    }
+}
diff --git a/src/org/antlr/runtime/CharStream.java b/src/org/antlr/runtime/CharStream.java
new file mode 100755
index 0000000..db55755
--- /dev/null
+++ b/src/org/antlr/runtime/CharStream.java
@@ -0,0 +1,57 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A source of characters for an ANTLR lexer */
+public interface CharStream extends IntStream {
+    public static final int EOF = -1;
+
+	/** For infinite streams, you don't need this; primarily I'm providing
+	 *  a useful interface for action code.  Just make sure actions don't
+	 *  use this on streams that don't support it.
+	 */
+	public String substring(int start, int stop);
+
+	/** Get the ith character of lookahead.  This is the same usually as
+	 *  LA(i).  This will be used for labels in the generated
+	 *  lexer code.  I'd prefer to return a char here type-wise, but it's
+	 *  probably better to be 32-bit clean and be consistent with LA.
+	 */
+	public int LT(int i);
+
+	/** ANTLR tracks the line information automatically */
+	int getLine();
+
+	/** Because this stream can rewind, we need to be able to reset the line */
+	void setLine(int line);
+
+	void setCharPositionInLine(int pos);
+
+	/** The index of the character relative to the beginning of the line 0..n-1 */
+	int getCharPositionInLine();
+}
diff --git a/src/org/antlr/runtime/CharStreamState.java b/src/org/antlr/runtime/CharStreamState.java
new file mode 100755
index 0000000..04d969a
--- /dev/null
+++ b/src/org/antlr/runtime/CharStreamState.java
@@ -0,0 +1,45 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** When walking ahead with cyclic DFA or for syntactic predicates,
+ *  we need to record the state of the input stream (char index,
+ *  line, etc...) so that we can rewind the state after scanning ahead.
+ *
+ *  This is the complete state of a stream.
+ */
+public class CharStreamState {
+	/** Index into the char stream of next lookahead char */
+	int p;
+
+	/** What line number is the scanner at before processing buffer[p]? */
+	int line;
+	
+	/** What char position 0..n-1 in line is scanner before processing buffer[p]? */
+	int charPositionInLine;
+}
diff --git a/src/org/antlr/runtime/ClassicToken.java b/src/org/antlr/runtime/ClassicToken.java
new file mode 100755
index 0000000..72c2bd9
--- /dev/null
+++ b/src/org/antlr/runtime/ClassicToken.java
@@ -0,0 +1,141 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A Token object like we'd use in ANTLR 2.x; has an actual string created
+ *  and associated with this object.  These objects are needed for imaginary
+ *  tree nodes that have payload objects.  We need to create a Token object
+ *  that has a string; the tree node will point at this token.  CommonToken
+ *  has indexes into a char stream and hence cannot be used to introduce
+ *  new strings.
+ */
+public class ClassicToken implements Token {
+	protected String text;
+	protected int type;
+	protected int line;
+	protected int charPositionInLine;
+	protected int channel=DEFAULT_CHANNEL;
+
+	/** What token number is this from 0..n-1 tokens */
+	protected int index;
+
+	public ClassicToken(int type) {
+		this.type = type;
+	}
+
+	public ClassicToken(Token oldToken) {
+		text = oldToken.getText();
+		type = oldToken.getType();
+		line = oldToken.getLine();
+		charPositionInLine = oldToken.getCharPositionInLine();
+		channel = oldToken.getChannel();
+	}
+
+	public ClassicToken(int type, String text) {
+		this.type = type;
+		this.text = text;
+	}
+
+	public ClassicToken(int type, String text, int channel) {
+		this.type = type;
+		this.text = text;
+		this.channel = channel;
+	}
+
+	public int getType() {
+		return type;
+	}
+
+	public void setLine(int line) {
+		this.line = line;
+	}
+
+	public String getText() {
+		return text;
+	}
+
+	public void setText(String text) {
+		this.text = text;
+	}
+
+	public int getLine() {
+		return line;
+	}
+
+	public int getCharPositionInLine() {
+		return charPositionInLine;
+	}
+
+	public void setCharPositionInLine(int charPositionInLine) {
+		this.charPositionInLine = charPositionInLine;
+	}
+
+	public int getChannel() {
+		return channel;
+	}
+
+	public void setChannel(int channel) {
+		this.channel = channel;
+	}
+
+	public void setType(int type) {
+		this.type = type;
+	}
+
+	public int getTokenIndex() {
+		return index;
+	}
+
+	public void setTokenIndex(int index) {
+		this.index = index;
+	}
+
+	public CharStream getInputStream() {
+		return null;
+	}
+
+	public void setInputStream(CharStream input) {
+	}
+	
+	public String toString() {
+		String channelStr = "";
+		if ( channel>0 ) {
+			channelStr=",channel="+channel;
+		}
+		String txt = getText();
+		if ( txt!=null ) {
+			txt = txt.replaceAll("\n","\\\\n");
+			txt = txt.replaceAll("\r","\\\\r");
+			txt = txt.replaceAll("\t","\\\\t");
+		}
+		else {
+			txt = "<no text>";
+		}
+		return "[@"+getTokenIndex()+",'"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
+	}
+}
diff --git a/src/org/antlr/runtime/CommonToken.java b/src/org/antlr/runtime/CommonToken.java
new file mode 100755
index 0000000..f01e1f4
--- /dev/null
+++ b/src/org/antlr/runtime/CommonToken.java
@@ -0,0 +1,191 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.io.Serializable;
+
+public class CommonToken implements Token, Serializable {
+	protected int type;
+	protected int line;
+	protected int charPositionInLine = -1; // set to invalid position
+	protected int channel=DEFAULT_CHANNEL;
+	protected transient CharStream input;
+
+	/** We need to be able to change the text once in a while.  If
+	 *  this is non-null, then getText should return this.  Note that
+	 *  start/stop are not affected by changing this.
+	  */
+	protected String text;
+
+	/** What token number is this from 0..n-1 tokens; < 0 implies invalid index */
+	protected int index = -1;
+
+	/** The char position into the input buffer where this token starts */
+	protected int start;
+
+	/** The char position into the input buffer where this token stops */
+	protected int stop;
+
+	public CommonToken(int type) {
+		this.type = type;
+	}
+
+	public CommonToken(CharStream input, int type, int channel, int start, int stop) {
+		this.input = input;
+		this.type = type;
+		this.channel = channel;
+		this.start = start;
+		this.stop = stop;
+	}
+
+	public CommonToken(int type, String text) {
+		this.type = type;
+		this.channel = DEFAULT_CHANNEL;
+		this.text = text;
+	}
+
+	public CommonToken(Token oldToken) {
+		text = oldToken.getText();
+		type = oldToken.getType();
+		line = oldToken.getLine();
+		index = oldToken.getTokenIndex();
+		charPositionInLine = oldToken.getCharPositionInLine();
+		channel = oldToken.getChannel();
+        input = oldToken.getInputStream();
+		if ( oldToken instanceof CommonToken ) {
+			start = ((CommonToken)oldToken).start;
+			stop = ((CommonToken)oldToken).stop;
+		}
+	}
+
+	public int getType() {
+		return type;
+	}
+
+	public void setLine(int line) {
+		this.line = line;
+	}
+
+	public String getText() {
+		if ( text!=null ) {
+			return text;
+		}
+		if ( input==null ) {
+			return null;
+		}
+        if ( start<input.size() && stop<input.size() ) {
+		    text = input.substring(start,stop);
+        }
+        else {
+            text = "<EOF>";
+        }
+		return text;
+	}
+
+	/** Override the text for this token.  getText() will return this text
+	 *  rather than pulling from the buffer.  Note that this does not mean
+	 *  that start/stop indexes are not valid.  It means that that input
+	 *  was converted to a new string in the token object.
+	 */
+	public void setText(String text) {
+		this.text = text;
+	}
+
+	public int getLine() {
+		return line;
+	}
+
+	public int getCharPositionInLine() {
+		return charPositionInLine;
+	}
+
+	public void setCharPositionInLine(int charPositionInLine) {
+		this.charPositionInLine = charPositionInLine;
+	}
+
+	public int getChannel() {
+		return channel;
+	}
+
+	public void setChannel(int channel) {
+		this.channel = channel;
+	}
+
+	public void setType(int type) {
+		this.type = type;
+	}
+
+	public int getStartIndex() {
+		return start;
+	}
+
+	public void setStartIndex(int start) {
+		this.start = start;
+	}
+
+	public int getStopIndex() {
+		return stop;
+	}
+
+	public void setStopIndex(int stop) {
+		this.stop = stop;
+	}
+
+	public int getTokenIndex() {
+		return index;
+	}
+
+	public void setTokenIndex(int index) {
+		this.index = index;
+	}
+
+	public CharStream getInputStream() {
+		return input;
+	}
+
+	public void setInputStream(CharStream input) {
+		this.input = input;
+	}
+
+	public String toString() {
+		String channelStr = "";
+		if ( channel>0 ) {
+			channelStr=",channel="+channel;
+		}
+		String txt = getText();
+		if ( txt!=null ) {
+			txt = txt.replaceAll("\n","\\\\n");
+			txt = txt.replaceAll("\r","\\\\r");
+			txt = txt.replaceAll("\t","\\\\t");
+		}
+		else {
+			txt = "<no text>";
+		}
+		return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
+	}
+}
diff --git a/src/org/antlr/runtime/CommonTokenStream.java b/src/org/antlr/runtime/CommonTokenStream.java
new file mode 100755
index 0000000..28135a6
--- /dev/null
+++ b/src/org/antlr/runtime/CommonTokenStream.java
@@ -0,0 +1,153 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime;
+
+/** The most common stream of tokens where every token is buffered up
+ *  and tokens are filtered for a certain channel (the parser will only
+ *  see these tokens).
+ *
+ *  Even though it buffers all of the tokens, this token stream pulls tokens
+ *  from the tokens source on demand. In other words, until you ask for a
+ *  token using consume(), LT(), etc. the stream does not pull from the lexer.
+ *
+ *  The only difference between this stream and BufferedTokenStream superclass
+ *  is that this stream knows how to ignore off channel tokens. There may be
+ *  a performance advantage to using the superclass if you don't pass
+ *  whitespace and comments etc. to the parser on a hidden channel (i.e.,
+ *  you set $channel instead of calling skip() in lexer rules.)
+ *
+ *  @see org.antlr.runtime.UnbufferedTokenStream
+ *  @see org.antlr.runtime.BufferedTokenStream
+ */
+public class CommonTokenStream extends BufferedTokenStream {
+    /** Skip tokens on any channel but this one; this is how we skip whitespace... */
+    protected int channel = Token.DEFAULT_CHANNEL;
+
+    public CommonTokenStream() { ; }
+
+    public CommonTokenStream(TokenSource tokenSource) {
+        super(tokenSource);
+    }
+
+    public CommonTokenStream(TokenSource tokenSource, int channel) {
+        this(tokenSource);
+        this.channel = channel;
+    }
+
+    /** Always leave p on an on-channel token. */
+    public void consume() {
+        if ( p == -1 ) setup();
+        p++;
+        sync(p);
+        while ( tokens.get(p).getChannel()!=channel ) {
+            p++;
+            sync(p);
+        }
+    }
+
+    protected Token LB(int k) {
+        if ( k==0 || (p-k)<0 ) return null;
+
+        int i = p;
+        int n = 1;
+        // find k good tokens looking backwards
+        while ( n<=k ) {
+            // skip off-channel tokens
+            i = skipOffTokenChannelsReverse(i-1);
+            n++;
+        }
+        if ( i<0 ) return null;
+        return tokens.get(i);
+    }
+
+    public Token LT(int k) {
+        //System.out.println("enter LT("+k+")");
+        if ( p == -1 ) setup();
+        if ( k == 0 ) return null;
+        if ( k < 0 ) return LB(-k);
+        int i = p;
+        int n = 1; // we know tokens[p] is a good one
+        // find k good tokens
+        while ( n<k ) {
+            // skip off-channel tokens
+            i = skipOffTokenChannels(i+1);
+            n++;
+        }
+		if ( i>range ) range = i;		
+        return tokens.get(i);
+    }
+
+    /** Given a starting index, return the index of the first on-channel
+     *  token.
+     */
+    protected int skipOffTokenChannels(int i) {
+        sync(i);
+        while ( tokens.get(i).getChannel()!=channel ) { // also stops at EOF (it's onchannel)
+            i++;
+            sync(i);
+        }
+        return i;
+    }
+
+    protected int skipOffTokenChannelsReverse(int i) {
+        while ( i>=0 && ((Token)tokens.get(i)).getChannel()!=channel ) {
+            i--;
+        }
+        return i;
+    }
+
+    protected void setup() {
+        p = 0;
+        sync(0);
+        int i = 0;
+        while ( tokens.get(i).getChannel()!=channel ) {
+            i++;
+            sync(i);
+        }
+        p = i;
+    }
+
+	/** Count EOF just once. */
+	public int getNumberOfOnChannelTokens() {
+		int n = 0;
+		fill();
+		for (int i = 0; i < tokens.size(); i++) {
+			Token t = tokens.get(i);
+			if ( t.getChannel()==channel ) n++;
+			if ( t.getType()==Token.EOF ) break;
+		}
+		return n;
+	}
+
+    /** Reset this token stream by setting its token source. */
+    public void setTokenSource(TokenSource tokenSource) {
+        super.setTokenSource(tokenSource);
+        channel = Token.DEFAULT_CHANNEL;
+    }
+}
diff --git a/src/org/antlr/runtime/DFA.java b/src/org/antlr/runtime/DFA.java
new file mode 100755
index 0000000..d1bebbc
--- /dev/null
+++ b/src/org/antlr/runtime/DFA.java
@@ -0,0 +1,229 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A DFA implemented as a set of transition tables.
+ *
+ *  Any state that has a semantic predicate edge is special; those states
+ *  are generated with if-then-else structures in a specialStateTransition()
+ *  which is generated by cyclicDFA template.
+ *
+ *  There are at most 32767 states (16-bit signed short).
+ *  Could get away with byte sometimes but would have to generate different
+ *  types and the simulation code too.  For a point of reference, the Java
+ *  lexer's Tokens rule DFA has 326 states roughly.
+ */
+public class DFA {
+	protected short[] eot;
+	protected short[] eof;
+	protected char[] min;
+    protected char[] max;
+    protected short[] accept;
+    protected short[] special;
+    protected short[][] transition;
+
+	protected int decisionNumber;
+
+	/** Which recognizer encloses this DFA?  Needed to check backtracking */
+	protected BaseRecognizer recognizer;
+
+	public static final boolean debug = false;
+
+	/** From the input stream, predict what alternative will succeed
+	 *  using this DFA (representing the covering regular approximation
+	 *  to the underlying CFL).  Return an alternative number 1..n.  Throw
+	 *  an exception upon error.
+	 */
+	public int predict(IntStream input)
+		throws RecognitionException
+	{
+		if ( debug ) {
+			System.err.println("Enter DFA.predict for decision "+decisionNumber);
+		}
+		int mark = input.mark(); // remember where decision started in input
+		int s = 0; // we always start at s0
+		try {
+			while ( true ) {
+				if ( debug ) System.err.println("DFA "+decisionNumber+" state "+s+" LA(1)="+(char)input.LA(1)+"("+input.LA(1)+
+												"), index="+input.index());
+				int specialState = special[s];
+				if ( specialState>=0 ) {
+					if ( debug ) {
+						System.err.println("DFA "+decisionNumber+
+							" state "+s+" is special state "+specialState);
+					}
+					s = specialStateTransition(specialState,input);
+					if ( debug ) {
+						System.err.println("DFA "+decisionNumber+
+							" returns from special state "+specialState+" to "+s);
+					}
+					if ( s==-1 ) {
+						noViableAlt(s,input);
+						return 0;
+					}
+					input.consume();
+					continue;
+				}
+				if ( accept[s] >= 1 ) {
+					if ( debug ) System.err.println("accept; predict "+accept[s]+" from state "+s);
+					return accept[s];
+				}
+				// look for a normal char transition
+				char c = (char)input.LA(1); // -1 == \uFFFF, all tokens fit in 65000 space
+				if (c>=min[s] && c<=max[s]) {
+					int snext = transition[s][c-min[s]]; // move to next state
+					if ( snext < 0 ) {
+						// was in range but not a normal transition
+						// must check EOT, which is like the else clause.
+						// eot[s]>=0 indicates that an EOT edge goes to another
+						// state.
+						if ( eot[s]>=0 ) {  // EOT Transition to accept state?
+							if ( debug ) System.err.println("EOT transition");
+							s = eot[s];
+							input.consume();
+							// TODO: I had this as return accept[eot[s]]
+							// which assumed here that the EOT edge always
+							// went to an accept...faster to do this, but
+							// what about predicated edges coming from EOT
+							// target?
+							continue;
+						}
+						noViableAlt(s,input);
+						return 0;
+					}
+					s = snext;
+					input.consume();
+					continue;
+				}
+				if ( eot[s]>=0 ) {  // EOT Transition?
+					if ( debug ) System.err.println("EOT transition");
+					s = eot[s];
+					input.consume();
+					continue;
+				}
+				if ( c==(char)Token.EOF && eof[s]>=0 ) {  // EOF Transition to accept state?
+					if ( debug ) System.err.println("accept via EOF; predict "+accept[eof[s]]+" from "+eof[s]);
+					return accept[eof[s]];
+				}
+				// not in range and not EOF/EOT, must be invalid symbol
+				if ( debug ) {
+					System.err.println("min["+s+"]="+min[s]);
+					System.err.println("max["+s+"]="+max[s]);
+					System.err.println("eot["+s+"]="+eot[s]);
+					System.err.println("eof["+s+"]="+eof[s]);
+					for (int p=0; p<transition[s].length; p++) {
+						System.err.print(transition[s][p]+" ");
+					}
+					System.err.println();
+				}
+				noViableAlt(s,input);
+				return 0;
+			}
+		}
+		finally {
+			input.rewind(mark);
+		}
+	}
+
+	protected void noViableAlt(int s, IntStream input) throws NoViableAltException {
+		if (recognizer.state.backtracking>0) {
+			recognizer.state.failed=true;
+			return;
+		}
+		NoViableAltException nvae =
+			new NoViableAltException(getDescription(),
+									 decisionNumber,
+									 s,
+									 input);
+		error(nvae);
+		throw nvae;
+	}
+
+	/** A hook for debugging interface */
+	protected void error(NoViableAltException nvae) { ; }
+
+	public int specialStateTransition(int s, IntStream input)
+		throws NoViableAltException
+	{
+		return -1;
+	}
+
+	public String getDescription() {
+		return "n/a";
+	}
+
+	/** Given a String that has a run-length-encoding of some unsigned shorts
+	 *  like "\1\2\3\9", convert to short[] {2,9,9,9}.  We do this to avoid
+	 *  static short[] which generates so much init code that the class won't
+	 *  compile. :(
+	 */
+	public static short[] unpackEncodedString(String encodedString) {
+		// walk first to find how big it is.
+		int size = 0;
+		for (int i=0; i<encodedString.length(); i+=2) {
+			size += encodedString.charAt(i);
+		}
+		short[] data = new short[size];
+		int di = 0;
+		for (int i=0; i<encodedString.length(); i+=2) {
+			char n = encodedString.charAt(i);
+			char v = encodedString.charAt(i+1);
+			// add v n times to data
+			for (int j=1; j<=n; j++) {
+				data[di++] = (short)v;
+			}
+		}
+		return data;
+	}
+
+	/** Hideous duplication of code, but I need different typed arrays out :( */
+	public static char[] unpackEncodedStringToUnsignedChars(String encodedString) {
+		// walk first to find how big it is.
+		int size = 0;
+		for (int i=0; i<encodedString.length(); i+=2) {
+			size += encodedString.charAt(i);
+		}
+		char[] data = new char[size];
+		int di = 0;
+		for (int i=0; i<encodedString.length(); i+=2) {
+			char n = encodedString.charAt(i);
+			char v = encodedString.charAt(i+1);
+			// add v n times to data
+			for (int j=1; j<=n; j++) {
+				data[di++] = v;
+			}
+		}
+		return data;
+	}
+
+	/*
+	public int specialTransition(int state, int symbol) {
+		return 0;
+	}
+	*/
+}
diff --git a/src/org/antlr/runtime/EarlyExitException.java b/src/org/antlr/runtime/EarlyExitException.java
new file mode 100755
index 0000000..1f9c1ec
--- /dev/null
+++ b/src/org/antlr/runtime/EarlyExitException.java
@@ -0,0 +1,41 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/**  The recognizer did not match anything for a (..)+ loop. */
+public class EarlyExitException extends RecognitionException {
+	public int decisionNumber;
+
+	/** Used for remote debugger deserialization */
+	public EarlyExitException() {;}
+	
+	public EarlyExitException(int decisionNumber, IntStream input) {
+		super(input);
+		this.decisionNumber = decisionNumber;
+	}
+}
diff --git a/src/org/antlr/runtime/FailedPredicateException.java b/src/org/antlr/runtime/FailedPredicateException.java
new file mode 100755
index 0000000..5bef1bd
--- /dev/null
+++ b/src/org/antlr/runtime/FailedPredicateException.java
@@ -0,0 +1,54 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A semantic predicate failed during validation.  Validation of predicates
+ *  occurs when normally parsing the alternative just like matching a token.
+ *  Disambiguating predicate evaluation occurs when we hoist a predicate into
+ *  a prediction decision.
+ */
+public class FailedPredicateException extends RecognitionException {
+	public String ruleName;
+	public String predicateText;
+
+	/** Used for remote debugger deserialization */
+	public FailedPredicateException() {;}
+
+	public FailedPredicateException(IntStream input,
+									String ruleName,
+									String predicateText)
+	{
+		super(input);
+		this.ruleName = ruleName;
+		this.predicateText = predicateText;
+	}
+
+	public String toString() {
+		return "FailedPredicateException("+ruleName+",{"+predicateText+"}?)";
+	}
+}
diff --git a/src/org/antlr/runtime/IntStream.java b/src/org/antlr/runtime/IntStream.java
new file mode 100755
index 0000000..01c7841
--- /dev/null
+++ b/src/org/antlr/runtime/IntStream.java
@@ -0,0 +1,122 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A simple stream of integers used when all I care about is the char
+ *  or token type sequence (such as interpretation).
+ */
+public interface IntStream {
+	void consume();
+
+	/** Get int at current input pointer + i ahead where i=1 is next int.
+	 *  Negative indexes are allowed.  LA(-1) is previous token (token
+	 *  just matched).  LA(-i) where i is before first token should
+	 *  yield -1, invalid char / EOF.
+	 */
+	int LA(int i);
+
+	/** Tell the stream to start buffering if it hasn't already.  Return
+     *  current input position, index(), or some other marker so that
+	 *  when passed to rewind() you get back to the same spot.
+	 *  rewind(mark()) should not affect the input cursor.  The Lexer
+	 *  track line/col info as well as input index so its markers are
+	 *  not pure input indexes.  Same for tree node streams.
+     */
+	int mark();
+
+	/** Return the current input symbol index 0..n where n indicates the
+     *  last symbol has been read.  The index is the symbol about to be
+	 *  read not the most recently read symbol.
+     */
+	int index();
+
+	/** Reset the stream so that next call to index would return marker.
+	 *  The marker will usually be index() but it doesn't have to be.  It's
+	 *  just a marker to indicate what state the stream was in.  This is
+	 *  essentially calling release() and seek().  If there are markers
+	 *  created after this marker argument, this routine must unroll them
+	 *  like a stack.  Assume the state the stream was in when this marker
+	 *  was created.
+	 */
+	void rewind(int marker);
+
+	/** Rewind to the input position of the last marker.
+	 *  Used currently only after a cyclic DFA and just
+	 *  before starting a sem/syn predicate to get the
+	 *  input position back to the start of the decision.
+	 *  Do not "pop" the marker off the state.  mark(i)
+	 *  and rewind(i) should balance still. It is
+	 *  like invoking rewind(last marker) but it should not "pop"
+	 *  the marker off.  It's like seek(last marker's input position).
+	 */
+	void rewind();
+
+	/** You may want to commit to a backtrack but don't want to force the
+	 *  stream to keep bookkeeping objects around for a marker that is
+	 *  no longer necessary.  This will have the same behavior as
+	 *  rewind() except it releases resources without the backward seek.
+	 *  This must throw away resources for all markers back to the marker
+	 *  argument.  So if you're nested 5 levels of mark(), and then release(2)
+	 *  you have to release resources for depths 2..5.
+	 */
+	void release(int marker);
+
+	/** Set the input cursor to the position indicated by index.  This is
+	 *  normally used to seek ahead in the input stream.  No buffering is
+	 *  required to do this unless you know your stream will use seek to
+	 *  move backwards such as when backtracking.
+	 *
+	 *  This is different from rewind in its multi-directional
+	 *  requirement and in that its argument is strictly an input cursor (index).
+	 *
+	 *  For char streams, seeking forward must update the stream state such
+	 *  as line number.  For seeking backwards, you will be presumably
+	 *  backtracking using the mark/rewind mechanism that restores state and
+	 *  so this method does not need to update state when seeking backwards.
+	 *
+	 *  Currently, this method is only used for efficient backtracking using
+	 *  memoization, but in the future it may be used for incremental parsing.
+	 *
+	 *  The index is 0..n-1.  A seek to position i means that LA(1) will
+	 *  return the ith symbol.  So, seeking to 0 means LA(1) will return the
+	 *  first element in the stream. 
+	 */
+	void seek(int index);
+
+	/** Only makes sense for streams that buffer everything up probably, but
+	 *  might be useful to display the entire stream or for testing.  This
+	 *  value includes a single EOF.
+	 */
+	int size();
+
+	/** Where are you getting symbols from?  Normally, implementations will
+	 *  pass the buck all the way to the lexer who can ask its input stream
+	 *  for the file name or whatever.
+	 */
+	public String getSourceName();
+}
diff --git a/src/org/antlr/runtime/LegacyCommonTokenStream.java b/src/org/antlr/runtime/LegacyCommonTokenStream.java
new file mode 100755
index 0000000..f9c5e39
--- /dev/null
+++ b/src/org/antlr/runtime/LegacyCommonTokenStream.java
@@ -0,0 +1,394 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.*;
+
+/** The most common stream of tokens is one where every token is buffered up
+ *  and tokens are prefiltered for a certain channel (the parser will only
+ *  see these tokens and cannot change the filter channel number during the
+ *  parse).
+ *
+ *  TODO: how to access the full token stream?  How to track all tokens matched per rule?
+ */
+public class LegacyCommonTokenStream implements TokenStream {
+    protected TokenSource tokenSource;
+
+	/** Record every single token pulled from the source so we can reproduce
+	 *  chunks of it later.
+	 */
+	protected List tokens;
+
+	/** Map<tokentype, channel> to override some Tokens' channel numbers */
+	protected Map channelOverrideMap;
+
+	/** Set<tokentype>; discard any tokens with this type */
+	protected Set discardSet;
+
+	/** Skip tokens on any channel but this one; this is how we skip whitespace... */
+	protected int channel = Token.DEFAULT_CHANNEL;
+
+	/** By default, track all incoming tokens */
+	protected boolean discardOffChannelTokens = false;
+
+	/** Track the last mark() call result value for use in rewind(). */
+	protected int lastMarker;
+
+	protected int range = -1; // how deep have we gone?	
+
+	/** The index into the tokens list of the current token (next token
+     *  to consume).  p==-1 indicates that the tokens list is empty
+     */
+    protected int p = -1;
+
+	public LegacyCommonTokenStream() {
+		tokens = new ArrayList(500);
+	}
+
+	public LegacyCommonTokenStream(TokenSource tokenSource) {
+	    this();
+		this.tokenSource = tokenSource;
+	}
+
+	public LegacyCommonTokenStream(TokenSource tokenSource, int channel) {
+		this(tokenSource);
+		this.channel = channel;
+	}
+
+	/** Reset this token stream by setting its token source. */
+	public void setTokenSource(TokenSource tokenSource) {
+		this.tokenSource = tokenSource;
+		tokens.clear();
+		p = -1;
+		channel = Token.DEFAULT_CHANNEL;
+	}
+
+	/** Load all tokens from the token source and put in tokens.
+	 *  This is done upon first LT request because you might want to
+	 *  set some token type / channel overrides before filling buffer.
+	 */
+	protected void fillBuffer() {
+		int index = 0;
+		Token t = tokenSource.nextToken();
+		while ( t!=null && t.getType()!=CharStream.EOF ) {
+			boolean discard = false;
+			// is there a channel override for token type?
+			if ( channelOverrideMap!=null ) {
+				Integer channelI = (Integer)
+					channelOverrideMap.get(new Integer(t.getType()));
+				if ( channelI!=null ) {
+					t.setChannel(channelI.intValue());
+				}
+			}
+			if ( discardSet!=null &&
+				 discardSet.contains(new Integer(t.getType())) )
+			{
+				discard = true;
+			}
+			else if ( discardOffChannelTokens && t.getChannel()!=this.channel ) {
+				discard = true;
+			}
+			if ( !discard )	{
+				t.setTokenIndex(index);
+				tokens.add(t);
+				index++;
+			}
+			t = tokenSource.nextToken();
+		}
+		// leave p pointing at first token on channel
+		p = 0;
+		p = skipOffTokenChannels(p);
+    }
+
+	/** Move the input pointer to the next incoming token.  The stream
+	 *  must become active with LT(1) available.  consume() simply
+	 *  moves the input pointer so that LT(1) points at the next
+	 *  input symbol. Consume at least one token.
+	 *
+	 *  Walk past any token not on the channel the parser is listening to.
+	 */
+	public void consume() {
+		if ( p<tokens.size() ) {
+            p++;
+			p = skipOffTokenChannels(p); // leave p on valid token
+        }
+    }
+
+	/** Given a starting index, return the index of the first on-channel
+	 *  token.
+	 */
+	protected int skipOffTokenChannels(int i) {
+		int n = tokens.size();
+		while ( i<n && ((Token)tokens.get(i)).getChannel()!=channel ) {
+			i++;
+		}
+		return i;
+	}
+
+	protected int skipOffTokenChannelsReverse(int i) {
+		while ( i>=0 && ((Token)tokens.get(i)).getChannel()!=channel ) {
+			i--;
+		}
+		return i;
+	}
+
+	/** A simple filter mechanism whereby you can tell this token stream
+	 *  to force all tokens of type ttype to be on channel.  For example,
+	 *  when interpreting, we cannot exec actions so we need to tell
+	 *  the stream to force all WS and NEWLINE to be a different, ignored
+	 *  channel.
+	 */
+	public void setTokenTypeChannel(int ttype, int channel) {
+		if ( channelOverrideMap==null ) {
+			channelOverrideMap = new HashMap();
+		}
+        channelOverrideMap.put(new Integer(ttype), new Integer(channel));
+	}
+
+	public void discardTokenType(int ttype) {
+		if ( discardSet==null ) {
+			discardSet = new HashSet();
+		}
+        discardSet.add(new Integer(ttype));
+	}
+
+	public void discardOffChannelTokens(boolean discardOffChannelTokens) {
+		this.discardOffChannelTokens = discardOffChannelTokens;
+	}
+
+	public List getTokens() {
+		if ( p == -1 ) {
+			fillBuffer();
+		}
+		return tokens;
+	}
+
+	public List getTokens(int start, int stop) {
+		return getTokens(start, stop, (BitSet)null);
+	}
+
+	/** Given a start and stop index, return a List of all tokens in
+	 *  the token type BitSet.  Return null if no tokens were found.  This
+	 *  method looks at both on and off channel tokens.
+	 */
+	public List getTokens(int start, int stop, BitSet types) {
+		if ( p == -1 ) {
+			fillBuffer();
+		}
+		if ( stop>=tokens.size() ) {
+			stop=tokens.size()-1;
+		}
+		if ( start<0 ) {
+			start=0;
+		}
+		if ( start>stop ) {
+			return null;
+		}
+
+		// list = tokens[start:stop]:{Token t, t.getType() in types}
+		List filteredTokens = new ArrayList();
+		for (int i=start; i<=stop; i++) {
+			Token t = (Token)tokens.get(i);
+			if ( types==null || types.member(t.getType()) ) {
+				filteredTokens.add(t);
+			}
+		}
+		if ( filteredTokens.size()==0 ) {
+			filteredTokens = null;
+		}
+		return filteredTokens;
+	}
+
+	public List getTokens(int start, int stop, List types) {
+		return getTokens(start,stop,new BitSet(types));
+	}
+
+	public List getTokens(int start, int stop, int ttype) {
+		return getTokens(start,stop,BitSet.of(ttype));
+	}
+
+	/** Get the ith token from the current position 1..n where k=1 is the
+	 *  first symbol of lookahead.
+	 */
+	public Token LT(int k) {
+		if ( p == -1 ) {
+			fillBuffer();
+		}
+		if ( k==0 ) {
+			return null;
+		}
+		if ( k<0 ) {
+			return LB(-k);
+		}
+		//System.out.print("LT(p="+p+","+k+")=");
+		if ( (p+k-1) >= tokens.size() ) {
+            return (Token)tokens.get(tokens.size()-1);
+		}
+		//System.out.println(tokens.get(p+k-1));
+		int i = p;
+		int n = 1;
+		// find k good tokens
+		while ( n<k ) {
+			// skip off-channel tokens
+			i = skipOffTokenChannels(i+1); // leave p on valid token
+			n++;
+		}
+		if ( i>=tokens.size() ) {
+            return (Token)tokens.get(tokens.size()-1); // must be EOF
+		}
+
+		if ( i>range ) range = i;
+        return (Token)tokens.get(i);
+    }
+
+	/** Look backwards k tokens on-channel tokens */
+	protected Token LB(int k) {
+		//System.out.print("LB(p="+p+","+k+") ");
+		if ( p == -1 ) {
+			fillBuffer();
+		}
+		if ( k==0 ) {
+			return null;
+		}
+		if ( (p-k)<0 ) {
+			return null;
+		}
+
+		int i = p;
+		int n = 1;
+		// find k good tokens looking backwards
+		while ( n<=k ) {
+			// skip off-channel tokens
+			i = skipOffTokenChannelsReverse(i-1); // leave p on valid token
+			n++;
+		}
+		if ( i<0 ) {
+			return null;
+		}
+		return (Token)tokens.get(i);
+	}
+
+	/** Return absolute token i; ignore which channel the tokens are on;
+	 *  that is, count all tokens not just on-channel tokens.
+	 */
+	public Token get(int i) {
+		return (Token)tokens.get(i);
+	}
+
+	/** Get all tokens from start..stop inclusively */
+	public List get(int start, int stop) {
+		if ( p == -1 ) fillBuffer();
+		if ( start<0 || stop<0 ) return null;
+		return tokens.subList(start, stop);
+	}
+
+	public int LA(int i) {
+        return LT(i).getType();
+    }
+
+    public int mark() {
+		if ( p == -1 ) {
+			fillBuffer();
+		}
+		lastMarker = index();
+		return lastMarker;
+	}
+
+	public void release(int marker) {
+		// no resources to release
+	}
+
+	public int size() {
+		return tokens.size();
+	}
+
+    public int index() {
+        return p;
+    }
+
+	public int range() {
+		return range;
+	}
+
+	public void rewind(int marker) {
+		seek(marker);
+	}
+
+	public void rewind() {
+		seek(lastMarker);
+	}
+
+	public void reset() {
+		p = 0;
+		lastMarker = 0;
+	}
+	
+	public void seek(int index) {
+		p = index;
+	}
+
+	public TokenSource getTokenSource() {
+		return tokenSource;
+	}
+
+	public String getSourceName() {
+		return getTokenSource().getSourceName();
+	}
+
+	public String toString() {
+		if ( p == -1 ) {
+			fillBuffer();
+		}
+		return toString(0, tokens.size()-1);
+	}
+
+	public String toString(int start, int stop) {
+		if ( start<0 || stop<0 ) {
+			return null;
+		}
+		if ( p == -1 ) {
+			fillBuffer();
+		}
+		if ( stop>=tokens.size() ) {
+			stop = tokens.size()-1;
+		}
+ 		StringBuffer buf = new StringBuffer();
+		for (int i = start; i <= stop; i++) {
+			Token t = (Token)tokens.get(i);
+			buf.append(t.getText());
+		}
+		return buf.toString();
+	}
+
+	public String toString(Token start, Token stop) {
+		if ( start!=null && stop!=null ) {
+			return toString(start.getTokenIndex(), stop.getTokenIndex());
+		}
+		return null;
+	}
+}
diff --git a/src/org/antlr/runtime/Lexer.java b/src/org/antlr/runtime/Lexer.java
new file mode 100755
index 0000000..6037bcf
--- /dev/null
+++ b/src/org/antlr/runtime/Lexer.java
@@ -0,0 +1,340 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A lexer is recognizer that draws input symbols from a character stream.
+ *  lexer grammars result in a subclass of this object. A Lexer object
+ *  uses simplified match() and error recovery mechanisms in the interest
+ *  of speed.
+ */
+public abstract class Lexer extends BaseRecognizer implements TokenSource {
+	/** Where is the lexer drawing characters from? */
+	protected CharStream input;
+
+	public Lexer() {
+	}
+
+	public Lexer(CharStream input) {
+		this.input = input;
+	}
+
+	public Lexer(CharStream input, RecognizerSharedState state) {
+		super(state);
+		this.input = input;
+	}
+
+	public void reset() {
+		super.reset(); // reset all recognizer state variables
+		// wack Lexer state variables
+		if ( input!=null ) {
+			input.seek(0); // rewind the input
+		}
+		if ( state==null ) {
+			return; // no shared state work to do
+		}
+		state.token = null;
+		state.type = Token.INVALID_TOKEN_TYPE;
+		state.channel = Token.DEFAULT_CHANNEL;
+		state.tokenStartCharIndex = -1;
+		state.tokenStartCharPositionInLine = -1;
+		state.tokenStartLine = -1;
+		state.text = null;
+	}
+
+	/** Return a token from this source; i.e., match a token on the char
+	 *  stream.
+	 */
+	public Token nextToken() {
+		while (true) {
+			state.token = null;
+			state.channel = Token.DEFAULT_CHANNEL;
+			state.tokenStartCharIndex = input.index();
+			state.tokenStartCharPositionInLine = input.getCharPositionInLine();
+			state.tokenStartLine = input.getLine();
+			state.text = null;
+			if ( input.LA(1)==CharStream.EOF ) {
+                Token eof = new CommonToken((CharStream)input,Token.EOF,
+                                            Token.DEFAULT_CHANNEL,
+                                            input.index(),input.index());
+                eof.setLine(getLine());
+                eof.setCharPositionInLine(getCharPositionInLine());
+                return eof;
+			}
+			try {
+				mTokens();
+				if ( state.token==null ) {
+					emit();
+				}
+				else if ( state.token==Token.SKIP_TOKEN ) {
+					continue;
+				}
+				return state.token;
+			}
+			catch (NoViableAltException nva) {
+				reportError(nva);
+				recover(nva); // throw out current char and try again
+			}
+			catch (RecognitionException re) {
+				reportError(re);
+				// match() routine has already called recover()
+			}
+		}
+	}
+
+	/** Instruct the lexer to skip creating a token for current lexer rule
+	 *  and look for another token.  nextToken() knows to keep looking when
+	 *  a lexer rule finishes with token set to SKIP_TOKEN.  Recall that
+	 *  if token==null at end of any token rule, it creates one for you
+	 *  and emits it.
+	 */
+	public void skip() {
+		state.token = Token.SKIP_TOKEN;
+	}
+
+	/** This is the lexer entry point that sets instance var 'token' */
+	public abstract void mTokens() throws RecognitionException;
+
+	/** Set the char stream and reset the lexer */
+	public void setCharStream(CharStream input) {
+		this.input = null;
+		reset();
+		this.input = input;
+	}
+
+	public CharStream getCharStream() {
+		return this.input;
+	}
+
+	public String getSourceName() {
+		return input.getSourceName();
+	}
+
+	/** Currently does not support multiple emits per nextToken invocation
+	 *  for efficiency reasons.  Subclass and override this method and
+	 *  nextToken (to push tokens into a list and pull from that list rather
+	 *  than a single variable as this implementation does).
+	 */
+	public void emit(Token token) {
+		state.token = token;
+	}
+
+	/** The standard method called to automatically emit a token at the
+	 *  outermost lexical rule.  The token object should point into the
+	 *  char buffer start..stop.  If there is a text override in 'text',
+	 *  use that to set the token's text.  Override this method to emit
+	 *  custom Token objects.
+	 *
+	 *  If you are building trees, then you should also override
+	 *  Parser or TreeParser.getMissingSymbol().
+	 */
+	public Token emit() {
+		Token t = new CommonToken(input, state.type, state.channel, state.tokenStartCharIndex, getCharIndex()-1);
+		t.setLine(state.tokenStartLine);
+		t.setText(state.text);
+		t.setCharPositionInLine(state.tokenStartCharPositionInLine);
+		emit(t);
+		return t;
+	}
+
+	public void match(String s) throws MismatchedTokenException {
+		int i = 0;
+		while ( i<s.length() ) {
+			if ( input.LA(1)!=s.charAt(i) ) {
+				if ( state.backtracking>0 ) {
+					state.failed = true;
+					return;
+				}
+				MismatchedTokenException mte =
+					new MismatchedTokenException(s.charAt(i), input);
+				recover(mte);
+				throw mte;
+			}
+			i++;
+			input.consume();
+			state.failed = false;
+		}
+	}
+
+	public void matchAny() {
+		input.consume();
+	}
+
+	public void match(int c) throws MismatchedTokenException {
+		if ( input.LA(1)!=c ) {
+			if ( state.backtracking>0 ) {
+				state.failed = true;
+				return;
+			}
+			MismatchedTokenException mte =
+				new MismatchedTokenException(c, input);
+			recover(mte);  // don't really recover; just consume in lexer
+			throw mte;
+		}
+		input.consume();
+		state.failed = false;
+	}
+
+	public void matchRange(int a, int b)
+		throws MismatchedRangeException
+	{
+		if ( input.LA(1)<a || input.LA(1)>b ) {
+			if ( state.backtracking>0 ) {
+				state.failed = true;
+				return;
+			}
+			MismatchedRangeException mre =
+				new MismatchedRangeException(a,b,input);
+			recover(mre);
+			throw mre;
+		}
+		input.consume();
+		state.failed = false;
+	}
+
+	public int getLine() {
+		return input.getLine();
+	}
+
+	public int getCharPositionInLine() {
+		return input.getCharPositionInLine();
+	}
+
+	/** What is the index of the current character of lookahead? */
+	public int getCharIndex() {
+		return input.index();
+	}
+
+	/** Return the text matched so far for the current token or any
+	 *  text override.
+	 */
+	public String getText() {
+		if ( state.text!=null ) {
+			return state.text;
+		}
+		return input.substring(state.tokenStartCharIndex,getCharIndex()-1);
+	}
+
+	/** Set the complete text of this token; it wipes any previous
+	 *  changes to the text.
+	 */
+	public void setText(String text) {
+		state.text = text;
+	}
+
+	public void reportError(RecognitionException e) {
+		/** TODO: not thought about recovery in lexer yet.
+		 *
+		// if we've already reported an error and have not matched a token
+		// yet successfully, don't report any errors.
+		if ( errorRecovery ) {
+			//System.err.print("[SPURIOUS] ");
+			return;
+		}
+		errorRecovery = true;
+		 */
+
+		displayRecognitionError(this.getTokenNames(), e);
+	}
+
+	public String getErrorMessage(RecognitionException e, String[] tokenNames) {
+		String msg = null;
+		if ( e instanceof MismatchedTokenException ) {
+			MismatchedTokenException mte = (MismatchedTokenException)e;
+			msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting "+getCharErrorDisplay(mte.expecting);
+		}
+		else if ( e instanceof NoViableAltException ) {
+			NoViableAltException nvae = (NoViableAltException)e;
+			// for development, can add "decision=<<"+nvae.grammarDecisionDescription+">>"
+			// and "(decision="+nvae.decisionNumber+") and
+			// "state "+nvae.stateNumber
+			msg = "no viable alternative at character "+getCharErrorDisplay(e.c);
+		}
+		else if ( e instanceof EarlyExitException ) {
+			EarlyExitException eee = (EarlyExitException)e;
+			// for development, can add "(decision="+eee.decisionNumber+")"
+			msg = "required (...)+ loop did not match anything at character "+getCharErrorDisplay(e.c);
+		}
+		else if ( e instanceof MismatchedNotSetException ) {
+			MismatchedNotSetException mse = (MismatchedNotSetException)e;
+			msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting;
+		}
+		else if ( e instanceof MismatchedSetException ) {
+			MismatchedSetException mse = (MismatchedSetException)e;
+			msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting;
+		}
+		else if ( e instanceof MismatchedRangeException ) {
+			MismatchedRangeException mre = (MismatchedRangeException)e;
+			msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+
+				  getCharErrorDisplay(mre.a)+".."+getCharErrorDisplay(mre.b);
+		}
+		else {
+			msg = super.getErrorMessage(e, tokenNames);
+		}
+		return msg;
+	}
+
+	public String getCharErrorDisplay(int c) {
+		String s = String.valueOf((char)c);
+		switch ( c ) {
+			case Token.EOF :
+				s = "<EOF>";
+				break;
+			case '\n' :
+				s = "\\n";
+				break;
+			case '\t' :
+				s = "\\t";
+				break;
+			case '\r' :
+				s = "\\r";
+				break;
+		}
+		return "'"+s+"'";
+	}
+
+	/** Lexers can normally match any char in it's vocabulary after matching
+	 *  a token, so do the easy thing and just kill a character and hope
+	 *  it all works out.  You can instead use the rule invocation stack
+	 *  to do sophisticated error recovery if you are in a fragment rule.
+	 */
+	public void recover(RecognitionException re) {
+		//System.out.println("consuming char "+(char)input.LA(1)+" during recovery");
+		//re.printStackTrace();
+		input.consume();
+	}
+
+	public void traceIn(String ruleName, int ruleIndex)  {
+		String inputSymbol = ((char)input.LT(1))+" line="+getLine()+":"+getCharPositionInLine();
+		super.traceIn(ruleName, ruleIndex, inputSymbol);
+	}
+
+	public void traceOut(String ruleName, int ruleIndex)  {
+		String inputSymbol = ((char)input.LT(1))+" line="+getLine()+":"+getCharPositionInLine();
+		super.traceOut(ruleName, ruleIndex, inputSymbol);
+	}
+}
diff --git a/src/org/antlr/runtime/MismatchedNotSetException.java b/src/org/antlr/runtime/MismatchedNotSetException.java
new file mode 100755
index 0000000..49ceb27
--- /dev/null
+++ b/src/org/antlr/runtime/MismatchedNotSetException.java
@@ -0,0 +1,41 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+public class MismatchedNotSetException extends MismatchedSetException {
+	/** Used for remote debugger deserialization */
+	public MismatchedNotSetException() {;}
+
+	public MismatchedNotSetException(BitSet expecting, IntStream input) {
+		super(expecting, input);
+	}
+
+	public String toString() {
+		return "MismatchedNotSetException("+getUnexpectedType()+"!="+expecting+")";
+	}
+}
diff --git a/src/org/antlr/runtime/MismatchedRangeException.java b/src/org/antlr/runtime/MismatchedRangeException.java
new file mode 100755
index 0000000..23b3d87
--- /dev/null
+++ b/src/org/antlr/runtime/MismatchedRangeException.java
@@ -0,0 +1,45 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+public class MismatchedRangeException extends RecognitionException {
+	public int a,b;
+
+	/** Used for remote debugger deserialization */
+	public MismatchedRangeException() {;}
+
+	public MismatchedRangeException(int a, int b, IntStream input) {
+		super(input);
+		this.a = a;
+		this.b = b;
+	}
+
+	public String toString() {
+		return "MismatchedNotSetException("+getUnexpectedType()+" not in ["+a+","+b+"])";
+	}
+}
diff --git a/src/org/antlr/runtime/MismatchedSetException.java b/src/org/antlr/runtime/MismatchedSetException.java
new file mode 100755
index 0000000..9bfa530
--- /dev/null
+++ b/src/org/antlr/runtime/MismatchedSetException.java
@@ -0,0 +1,44 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+public class MismatchedSetException extends RecognitionException {
+	public BitSet expecting;
+
+	/** Used for remote debugger deserialization */
+	public MismatchedSetException() {;}
+
+	public MismatchedSetException(BitSet expecting, IntStream input) {
+		super(input);
+		this.expecting = expecting;
+	}
+
+	public String toString() {
+		return "MismatchedSetException("+getUnexpectedType()+"!="+expecting+")";
+	}
+}
diff --git a/src/org/antlr/runtime/MismatchedTokenException.java b/src/org/antlr/runtime/MismatchedTokenException.java
new file mode 100755
index 0000000..07ae814
--- /dev/null
+++ b/src/org/antlr/runtime/MismatchedTokenException.java
@@ -0,0 +1,45 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A mismatched char or Token or tree node */
+public class MismatchedTokenException extends RecognitionException {
+	public int expecting = Token.INVALID_TOKEN_TYPE;
+
+	/** Used for remote debugger deserialization */
+	public MismatchedTokenException() {;}
+
+	public MismatchedTokenException(int expecting, IntStream input) {
+		super(input);
+		this.expecting = expecting;
+	}
+
+	public String toString() {
+		return "MismatchedTokenException("+getUnexpectedType()+"!="+expecting+")";
+	}
+}
diff --git a/src/org/antlr/runtime/MismatchedTreeNodeException.java b/src/org/antlr/runtime/MismatchedTreeNodeException.java
new file mode 100755
index 0000000..99c834d
--- /dev/null
+++ b/src/org/antlr/runtime/MismatchedTreeNodeException.java
@@ -0,0 +1,49 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import org.antlr.runtime.tree.TreeNodeStream;
+import org.antlr.runtime.tree.Tree;
+
+/**
+ */
+public class MismatchedTreeNodeException extends RecognitionException {
+	public int expecting;
+
+	public MismatchedTreeNodeException() {
+	}
+
+	public MismatchedTreeNodeException(int expecting, TreeNodeStream input) {
+		super(input);
+		this.expecting = expecting;
+	}
+
+	public String toString() {
+		return "MismatchedTreeNodeException("+getUnexpectedType()+"!="+expecting+")";
+	}
+}
diff --git a/src/org/antlr/runtime/MissingTokenException.java b/src/org/antlr/runtime/MissingTokenException.java
new file mode 100755
index 0000000..9eda1f2
--- /dev/null
+++ b/src/org/antlr/runtime/MissingTokenException.java
@@ -0,0 +1,56 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** We were expecting a token but it's not found.  The current token
+ *  is actually what we wanted next.  Used for tree node errors too.
+ */
+public class MissingTokenException extends MismatchedTokenException {
+	public Object inserted;
+	/** Used for remote debugger deserialization */
+	public MissingTokenException() {;}
+
+	public MissingTokenException(int expecting, IntStream input, Object inserted) {
+		super(expecting, input);
+		this.inserted = inserted;
+	}
+
+	public int getMissingType() {
+		return expecting;
+	}
+
+	public String toString() {
+		if ( inserted!=null && token!=null ) {
+			return "MissingTokenException(inserted "+inserted+" at "+token.getText()+")";
+		}
+		if ( token!=null ) {
+			return "MissingTokenException(at "+token.getText()+")";
+		}
+		return "MissingTokenException";
+	}
+}
diff --git a/src/org/antlr/runtime/NoViableAltException.java b/src/org/antlr/runtime/NoViableAltException.java
new file mode 100755
index 0000000..889045f
--- /dev/null
+++ b/src/org/antlr/runtime/NoViableAltException.java
@@ -0,0 +1,57 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+public class NoViableAltException extends RecognitionException {
+	public String grammarDecisionDescription;
+	public int decisionNumber;
+	public int stateNumber;
+
+	/** Used for remote debugger deserialization */
+	public NoViableAltException() {;}
+	
+	public NoViableAltException(String grammarDecisionDescription,
+								int decisionNumber,
+								int stateNumber,
+								IntStream input)
+	{
+		super(input);
+		this.grammarDecisionDescription = grammarDecisionDescription;
+		this.decisionNumber = decisionNumber;
+		this.stateNumber = stateNumber;
+	}
+
+	public String toString() {
+		if ( input instanceof CharStream ) {
+			return "NoViableAltException('"+(char)getUnexpectedType()+"'@["+grammarDecisionDescription+"])";
+		}
+		else {
+			return "NoViableAltException("+getUnexpectedType()+"@["+grammarDecisionDescription+"])";
+		}
+	}
+}
diff --git a/src/org/antlr/runtime/Parser.java b/src/org/antlr/runtime/Parser.java
new file mode 100755
index 0000000..9cf937a
--- /dev/null
+++ b/src/org/antlr/runtime/Parser.java
@@ -0,0 +1,98 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A parser for TokenStreams.  "parser grammars" result in a subclass
+ *  of this.
+ */
+public class Parser extends BaseRecognizer {
+	public TokenStream input;
+
+	public Parser(TokenStream input) {
+		super(); // highlight that we go to super to set state object
+		setTokenStream(input);
+    }
+
+	public Parser(TokenStream input, RecognizerSharedState state) {
+		super(state); // share the state object with another parser
+		this.input = input;
+    }
+
+	public void reset() {
+		super.reset(); // reset all recognizer state variables
+		if ( input!=null ) {
+			input.seek(0); // rewind the input
+		}
+	}
+
+	protected Object getCurrentInputSymbol(IntStream input) {
+		return ((TokenStream)input).LT(1);
+	}
+
+	protected Object getMissingSymbol(IntStream input,
+									  RecognitionException e,
+									  int expectedTokenType,
+									  BitSet follow)
+	{
+		String tokenText = null;
+		if ( expectedTokenType==Token.EOF ) tokenText = "<missing EOF>";
+		else tokenText = "<missing "+getTokenNames()[expectedTokenType]+">";
+		CommonToken t = new CommonToken(expectedTokenType, tokenText);
+		Token current = ((TokenStream)input).LT(1);
+		if ( current.getType() == Token.EOF ) {
+			current = ((TokenStream)input).LT(-1);
+		}
+		t.line = current.getLine();
+		t.charPositionInLine = current.getCharPositionInLine();
+		t.channel = DEFAULT_TOKEN_CHANNEL;
+		return t;
+	}
+
+	/** Set the token stream and reset the parser */
+	public void setTokenStream(TokenStream input) {
+		this.input = null;
+		reset();
+		this.input = input;
+	}
+
+    public TokenStream getTokenStream() {
+		return input;
+	}
+
+	public String getSourceName() {
+		return input.getSourceName();
+	}
+
+	public void traceIn(String ruleName, int ruleIndex)  {
+		super.traceIn(ruleName, ruleIndex, input.LT(1));
+	}
+
+	public void traceOut(String ruleName, int ruleIndex)  {
+		super.traceOut(ruleName, ruleIndex, input.LT(1));
+	}
+}
diff --git a/src/org/antlr/runtime/ParserRuleReturnScope.java b/src/org/antlr/runtime/ParserRuleReturnScope.java
new file mode 100755
index 0000000..9237e4e
--- /dev/null
+++ b/src/org/antlr/runtime/ParserRuleReturnScope.java
@@ -0,0 +1,52 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** Rules that return more than a single value must return an object
+ *  containing all the values.  Besides the properties defined in
+ *  RuleLabelScope.predefinedRulePropertiesScope there may be user-defined
+ *  return values.  This class simply defines the minimum properties that
+ *  are always defined and methods to access the others that might be
+ *  available depending on output option such as template and tree.
+ *
+ *  Note text is not an actual property of the return value, it is computed
+ *  from start and stop using the input stream's toString() method.  I
+ *  could add a ctor to this so that we can pass in and store the input
+ *  stream, but I'm not sure we want to do that.  It would seem to be undefined
+ *  to get the .text property anyway if the rule matches tokens from multiple
+ *  input streams.
+ *
+ *  I do not use getters for fields of objects that are used simply to
+ *  group values such as this aggregate.  The getters/setters are there to
+ *  satisfy the superclass interface.
+ */
+public class ParserRuleReturnScope extends RuleReturnScope {
+	public Token start, stop;
+	public Object getStart() { return start; }
+	public Object getStop() { return stop; }
+}
diff --git a/src/org/antlr/runtime/RecognitionException.java b/src/org/antlr/runtime/RecognitionException.java
new file mode 100755
index 0000000..3e79f99
--- /dev/null
+++ b/src/org/antlr/runtime/RecognitionException.java
@@ -0,0 +1,180 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import org.antlr.runtime.tree.*;
+
+/** The root of the ANTLR exception hierarchy.
+ *
+ *  To avoid English-only error messages and to generally make things
+ *  as flexible as possible, these exceptions are not created with strings,
+ *  but rather the information necessary to generate an error.  Then
+ *  the various reporting methods in Parser and Lexer can be overridden
+ *  to generate a localized error message.  For example, MismatchedToken
+ *  exceptions are built with the expected token type.
+ *  So, don't expect getMessage() to return anything.
+ *
+ *  Note that as of Java 1.4, you can access the stack trace, which means
+ *  that you can compute the complete trace of rules from the start symbol.
+ *  This gives you considerable context information with which to generate
+ *  useful error messages.
+ *
+ *  ANTLR generates code that throws exceptions upon recognition error and
+ *  also generates code to catch these exceptions in each rule.  If you
+ *  want to quit upon first error, you can turn off the automatic error
+ *  handling mechanism using rulecatch action, but you still need to
+ *  override methods mismatch and recoverFromMismatchSet.
+ *
+ *  In general, the recognition exceptions can track where in a grammar a
+ *  problem occurred and/or what was the expected input.  While the parser
+ *  knows its state (such as current input symbol and line info) that
+ *  state can change before the exception is reported so current token index
+ *  is computed and stored at exception time.  From this info, you can
+ *  perhaps print an entire line of input not just a single token, for example.
+ *  Better to just say the recognizer had a problem and then let the parser
+ *  figure out a fancy report.
+ */
+public class RecognitionException extends Exception {
+	/** What input stream did the error occur in? */
+	public transient IntStream input;
+
+	/** What is index of token/char were we looking at when the error occurred? */
+	public int index;
+
+	/** The current Token when an error occurred.  Since not all streams
+	 *  can retrieve the ith Token, we have to track the Token object.
+	 *  For parsers.  Even when it's a tree parser, token might be set.
+	 */
+	public Token token;
+
+	/** If this is a tree parser exception, node is set to the node with
+	 *  the problem.
+	 */
+	public Object node;
+
+	/** The current char when an error occurred. For lexers. */
+	public int c;
+
+	/** Track the line at which the error occurred in case this is
+	 *  generated from a lexer.  We need to track this since the
+	 *  unexpected char doesn't carry the line info.
+	 */
+	public int line;
+
+	public int charPositionInLine;
+
+	/** If you are parsing a tree node stream, you will encounter som
+	 *  imaginary nodes w/o line/col info.  We now search backwards looking
+	 *  for most recent token with line/col info, but notify getErrorHeader()
+	 *  that info is approximate.
+	 */
+	public boolean approximateLineInfo;
+
+	/** Used for remote debugger deserialization */
+	public RecognitionException() {
+	}
+
+	public RecognitionException(IntStream input) {
+		this.input = input;
+		this.index = input.index();
+		if ( input instanceof TokenStream ) {
+			this.token = ((TokenStream)input).LT(1);
+			this.line = token.getLine();
+			this.charPositionInLine = token.getCharPositionInLine();
+		}
+		if ( input instanceof TreeNodeStream ) {
+			extractInformationFromTreeNodeStream(input);
+		}
+		else if ( input instanceof CharStream ) {
+			this.c = input.LA(1);
+			this.line = ((CharStream)input).getLine();
+			this.charPositionInLine = ((CharStream)input).getCharPositionInLine();
+		}
+		else {
+			this.c = input.LA(1);
+		}
+	}
+
+	protected void extractInformationFromTreeNodeStream(IntStream input) {
+		TreeNodeStream nodes = (TreeNodeStream)input;
+		this.node = nodes.LT(1);
+		TreeAdaptor adaptor = nodes.getTreeAdaptor();
+		Token payload = adaptor.getToken(node);
+		if ( payload!=null ) {
+			this.token = payload;
+			if ( payload.getLine()<= 0 ) {
+				// imaginary node; no line/pos info; scan backwards
+				int i = -1;
+				Object priorNode = nodes.LT(i);
+				while ( priorNode!=null ) {
+					Token priorPayload = adaptor.getToken(priorNode);
+					if ( priorPayload!=null && priorPayload.getLine()>0 ) {
+						// we found the most recent real line / pos info
+						this.line = priorPayload.getLine();
+						this.charPositionInLine = priorPayload.getCharPositionInLine();
+						this.approximateLineInfo = true;
+						break;
+					}
+					--i;
+					priorNode = nodes.LT(i);
+				}
+			}
+			else { // node created from real token
+				this.line = payload.getLine();
+				this.charPositionInLine = payload.getCharPositionInLine();
+			}
+		}
+		else if ( this.node instanceof Tree) {
+			this.line = ((Tree)this.node).getLine();
+			this.charPositionInLine = ((Tree)this.node).getCharPositionInLine();
+			if ( this.node instanceof CommonTree) {
+				this.token = ((CommonTree)this.node).token;
+			}
+		}
+		else {
+			int type = adaptor.getType(this.node);
+			String text = adaptor.getText(this.node);
+			this.token = new CommonToken(type, text);
+		}
+	}
+
+	/** Return the token type or char of the unexpected input element */
+	public int getUnexpectedType() {
+		if ( input instanceof TokenStream ) {
+			return token.getType();
+		}
+		else if ( input instanceof TreeNodeStream ) {
+			TreeNodeStream nodes = (TreeNodeStream)input;
+			TreeAdaptor adaptor = nodes.getTreeAdaptor();
+			return adaptor.getType(node);
+		}
+		else {
+			return c;
+		}
+	}
+}
diff --git a/src/org/antlr/runtime/RecognizerSharedState.java b/src/org/antlr/runtime/RecognizerSharedState.java
new file mode 100755
index 0000000..068ac3b
--- /dev/null
+++ b/src/org/antlr/runtime/RecognizerSharedState.java
@@ -0,0 +1,144 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */package org.antlr.runtime;
+
+import java.util.Map;
+
+/** The set of fields needed by an abstract recognizer to recognize input
+ *  and recover from errors etc...  As a separate state object, it can be
+ *  shared among multiple grammars; e.g., when one grammar imports another.
+ *
+ *  These fields are publically visible but the actual state pointer per
+ *  parser is protected.
+ */
+public class RecognizerSharedState {
+	/** Track the set of token types that can follow any rule invocation.
+	 *  Stack grows upwards.  When it hits the max, it grows 2x in size
+	 *  and keeps going.
+	 */
+	public BitSet[] following = new BitSet[BaseRecognizer.INITIAL_FOLLOW_STACK_SIZE];
+	public int _fsp = -1;
+
+	/** This is true when we see an error and before having successfully
+	 *  matched a token.  Prevents generation of more than one error message
+	 *  per error.
+	 */
+	public boolean errorRecovery = false;
+
+	/** The index into the input stream where the last error occurred.
+	 * 	This is used to prevent infinite loops where an error is found
+	 *  but no token is consumed during recovery...another error is found,
+	 *  ad naseum.  This is a failsafe mechanism to guarantee that at least
+	 *  one token/tree node is consumed for two errors.
+	 */
+	public int lastErrorIndex = -1;
+
+	/** In lieu of a return value, this indicates that a rule or token
+	 *  has failed to match.  Reset to false upon valid token match.
+	 */
+	public boolean failed = false;
+
+	/** Did the recognizer encounter a syntax error?  Track how many. */
+	public int syntaxErrors = 0;
+
+	/** If 0, no backtracking is going on.  Safe to exec actions etc...
+	 *  If >0 then it's the level of backtracking.
+	 */
+	public int backtracking = 0;
+
+	/** An array[size num rules] of Map<Integer,Integer> that tracks
+	 *  the stop token index for each rule.  ruleMemo[ruleIndex] is
+	 *  the memoization table for ruleIndex.  For key ruleStartIndex, you
+	 *  get back the stop token for associated rule or MEMO_RULE_FAILED.
+	 *
+	 *  This is only used if rule memoization is on (which it is by default).
+	 */
+	public Map[] ruleMemo;
+
+
+	// LEXER FIELDS (must be in same state object to avoid casting
+	//               constantly in generated code and Lexer object) :(
+
+
+	/** The goal of all lexer rules/methods is to create a token object.
+	 *  This is an instance variable as multiple rules may collaborate to
+	 *  create a single token.  nextToken will return this object after
+	 *  matching lexer rule(s).  If you subclass to allow multiple token
+	 *  emissions, then set this to the last token to be matched or
+	 *  something nonnull so that the auto token emit mechanism will not
+	 *  emit another token.
+	 */
+    public Token token;
+
+	/** What character index in the stream did the current token start at?
+	 *  Needed, for example, to get the text for current token.  Set at
+	 *  the start of nextToken.
+ 	 */
+	public int tokenStartCharIndex = -1;
+
+	/** The line on which the first character of the token resides */
+	public int tokenStartLine;
+
+	/** The character position of first character within the line */
+	public int tokenStartCharPositionInLine;
+
+	/** The channel number for the current token */
+	public int channel;
+
+	/** The token type for the current token */
+	public int type;
+
+	/** You can set the text for the current token to override what is in
+	 *  the input char buffer.  Use setText() or can set this instance var.
+ 	 */
+	public String text;
+
+    public RecognizerSharedState() {;}
+    
+    public RecognizerSharedState(RecognizerSharedState state) {
+        if ( this.following.length < state.following.length ) {
+            this.following = new BitSet[state.following.length];
+        }
+        System.arraycopy(state.following, 0, this.following, 0, state.following.length);
+        this._fsp = state._fsp;
+        this.errorRecovery = state.errorRecovery;
+        this.lastErrorIndex = state.lastErrorIndex;
+        this.failed = state.failed;
+        this.syntaxErrors = state.syntaxErrors;
+        this.backtracking = state.backtracking;
+        if ( state.ruleMemo!=null ) {
+            this.ruleMemo = new Map[state.ruleMemo.length];
+            System.arraycopy(state.ruleMemo, 0, this.ruleMemo, 0, state.ruleMemo.length);
+        }
+        this.token = state.token;
+        this.tokenStartCharIndex = state.tokenStartCharIndex;
+        this.tokenStartCharPositionInLine = state.tokenStartCharPositionInLine;
+        this.channel = state.channel;
+        this.type = state.type;
+        this.text = state.text;
+    }
+}
diff --git a/src/org/antlr/runtime/RuleReturnScope.java b/src/org/antlr/runtime/RuleReturnScope.java
new file mode 100755
index 0000000..85702ca
--- /dev/null
+++ b/src/org/antlr/runtime/RuleReturnScope.java
@@ -0,0 +1,42 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** Rules can return start/stop info as well as possible trees and templates */
+public class RuleReturnScope {
+	/** Return the start token or tree */
+	public Object getStart() { return null; }
+	/** Return the stop token or tree */
+	public Object getStop() { return null; }
+	/** Has a value potentially if output=AST; */
+	public Object getTree() { return null; }
+	/** Has a value potentially if output=template; Don't use StringTemplate
+	 *  type as it then causes a dependency with ST lib.
+	 */
+	public Object getTemplate() { return null; }
+}
diff --git a/src/org/antlr/runtime/SerializedGrammar.java b/src/org/antlr/runtime/SerializedGrammar.java
new file mode 100755
index 0000000..a609053
--- /dev/null
+++ b/src/org/antlr/runtime/SerializedGrammar.java
@@ -0,0 +1,198 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime;
+
+import java.io.IOException;
+import java.io.FileInputStream;
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.util.List;
+import java.util.ArrayList;
+
+public class SerializedGrammar {
+    public static final String COOKIE = "$ANTLR";
+    public static final int FORMAT_VERSION = 1;
+    //public static org.antlr.tool.Grammar gr; // TESTING ONLY; remove later
+
+    public String name;
+    public char type; // in {l, p, t, c}
+    public List rules;
+
+    class Rule {
+        String name;
+        Block block;
+        public Rule(String name, Block block) {
+            this.name = name;
+            this.block = block;
+        }
+        public String toString() {
+            return name+":"+block;
+        }
+    }
+
+    class Block {
+        List[] alts;
+        public Block(List[] alts) {
+            this.alts = alts;
+        }
+        public String toString() {
+            StringBuffer buf = new StringBuffer();
+            buf.append("(");
+            for (int i = 0; i < alts.length; i++) {
+                List alt = alts[i];
+                if ( i>0 ) buf.append("|");
+                buf.append(alt.toString());
+            }
+            buf.append(")");
+            return buf.toString();
+        }
+    }
+
+    class TokenRef {
+        int ttype;
+        public TokenRef(int ttype) { this.ttype = ttype; }
+        public String toString() { return String.valueOf(ttype); }
+    }
+
+    class RuleRef {
+        int ruleIndex;
+        public RuleRef(int ruleIndex) { this.ruleIndex = ruleIndex; }
+        public String toString() { return String.valueOf(ruleIndex); }
+    }
+
+    public SerializedGrammar(String filename) throws IOException {
+        System.out.println("loading "+filename);
+        FileInputStream fis = new FileInputStream(filename);
+        BufferedInputStream bos = new BufferedInputStream(fis);
+        DataInputStream in = new DataInputStream(bos);
+        readFile(in);
+        in.close();
+    }
+
+    protected void readFile(DataInputStream in) throws IOException {
+        String cookie = readString(in); // get $ANTLR
+        if ( !cookie.equals(COOKIE) ) throw new IOException("not a serialized grammar file");
+        int version = in.readByte();
+        char grammarType = (char)in.readByte();
+        this.type = grammarType;
+        String grammarName = readString(in);
+        this.name = grammarName;
+        System.out.println(grammarType+" grammar "+grammarName);
+        int numRules = in.readShort();
+        System.out.println("num rules = "+numRules);
+        rules = readRules(in, numRules);
+    }
+
+    protected List readRules(DataInputStream in, int numRules) throws IOException {
+        List rules = new ArrayList();
+        for (int i=0; i<numRules; i++) {
+            Rule r = readRule(in);
+            rules.add(r);
+        }
+        return rules;
+    }
+
+    protected Rule readRule(DataInputStream in) throws IOException {
+        byte R = in.readByte();
+        if ( R!='R' ) throw new IOException("missing R on start of rule");
+        String name = readString(in);
+        System.out.println("rule: "+name);
+        byte B = in.readByte();
+        Block b = readBlock(in);
+        byte period = in.readByte();
+        if ( period!='.' ) throw new IOException("missing . on end of rule");
+        return new Rule(name, b);
+    }
+
+    protected Block readBlock(DataInputStream in) throws IOException {
+        int nalts = in.readShort();
+        List[] alts = new List[nalts];
+        //System.out.println("enter block n="+nalts);
+        for (int i=0; i<nalts; i++) {
+            List alt = readAlt(in);
+            alts[i] = alt;
+        }
+        //System.out.println("exit block");
+        return new Block(alts);
+    }
+
+    protected List readAlt(DataInputStream in) throws IOException {
+        List alt = new ArrayList();
+        byte A = in.readByte();
+        if ( A!='A' ) throw new IOException("missing A on start of alt");
+        byte cmd = in.readByte();
+        while ( cmd!=';' ) {
+            switch (cmd) {
+                case 't' :
+                    int ttype = in.readShort();
+                    alt.add(new TokenRef(ttype));
+                    //System.out.println("read token "+gr.getTokenDisplayName(ttype));
+                    break;
+                case 'r' :
+                    int ruleIndex = in.readShort();
+                    alt.add(new RuleRef(ruleIndex));
+                    //System.out.println("read rule "+gr.getRuleName(ruleIndex));
+                    break;
+                case '.' : // wildcard
+                    break;
+                case '-' : // range
+                    int from = in.readChar();
+                    int to = in.readChar();
+                    break;
+                case '~' : // not
+                    int notThisTokenType = in.readShort();
+                    break;
+                case 'B' : // nested block
+                    Block b = readBlock(in);
+                    alt.add(b);
+                    break;
+            }
+            cmd = in.readByte();
+        }
+        //System.out.println("exit alt");
+        return alt;
+    }
+
+    protected String readString(DataInputStream in) throws IOException {
+        byte c = in.readByte();
+        StringBuffer buf = new StringBuffer();
+        while ( c!=';' ) {
+            buf.append((char)c);
+            c = in.readByte();
+        }
+        return buf.toString();
+    }
+
+    public String toString() {
+        StringBuffer buf = new StringBuffer();
+        buf.append(type+" grammar "+name);
+        buf.append(rules);
+        return buf.toString();
+    }
+}
diff --git a/src/org/antlr/runtime/Token.java b/src/org/antlr/runtime/Token.java
new file mode 100755
index 0000000..b8eb95e
--- /dev/null
+++ b/src/org/antlr/runtime/Token.java
@@ -0,0 +1,92 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+public interface Token {
+	public static final int EOR_TOKEN_TYPE = 1;
+
+	/** imaginary tree navigation type; traverse "get child" link */
+	public static final int DOWN = 2;
+	/** imaginary tree navigation type; finish with a child list */
+	public static final int UP = 3;
+
+	public static final int MIN_TOKEN_TYPE = UP+1;
+
+    public static final int EOF = CharStream.EOF;
+    // TODO: remove once we go ANTLR v3.3
+    public static final Token EOF_TOKEN = new CommonToken(EOF);
+
+	public static final int INVALID_TOKEN_TYPE = 0;
+	public static final Token INVALID_TOKEN = new CommonToken(INVALID_TOKEN_TYPE);
+
+	/** In an action, a lexer rule can set token to this SKIP_TOKEN and ANTLR
+	 *  will avoid creating a token for this symbol and try to fetch another.
+	 */
+	public static final Token SKIP_TOKEN = new CommonToken(INVALID_TOKEN_TYPE);
+
+	/** All tokens go to the parser (unless skip() is called in that rule)
+	 *  on a particular "channel".  The parser tunes to a particular channel
+	 *  so that whitespace etc... can go to the parser on a "hidden" channel.
+	 */
+	public static final int DEFAULT_CHANNEL = 0;
+	
+	/** Anything on different channel than DEFAULT_CHANNEL is not parsed
+	 *  by parser.
+	 */
+	public static final int HIDDEN_CHANNEL = 99;
+
+	/** Get the text of the token */
+	public String getText();
+	public void setText(String text);
+
+	public int getType();
+	public void setType(int ttype);
+	/**  The line number on which this token was matched; line=1..n */
+	public int getLine();
+    public void setLine(int line);
+
+	/** The index of the first character relative to the beginning of the line 0..n-1 */
+	public int getCharPositionInLine();
+	public void setCharPositionInLine(int pos);
+
+	public int getChannel();
+	public void setChannel(int channel);
+
+	/** An index from 0..n-1 of the token object in the input stream.
+	 *  This must be valid in order to use the ANTLRWorks debugger.
+	 */
+	public int getTokenIndex();
+	public void setTokenIndex(int index);
+
+	/** From what character stream was this token created?  You don't have to
+	 *  implement but it's nice to know where a Token comes from if you have
+	 *  include files etc... on the input.
+	 */
+	public CharStream getInputStream();
+	public void setInputStream(CharStream input);
+}
diff --git a/src/org/antlr/runtime/TokenRewriteStream.java b/src/org/antlr/runtime/TokenRewriteStream.java
new file mode 100755
index 0000000..8437441
--- /dev/null
+++ b/src/org/antlr/runtime/TokenRewriteStream.java
@@ -0,0 +1,590 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.*;
+
+/** Useful for dumping out the input stream after doing some
+ *  augmentation or other manipulations.
+ *
+ *  You can insert stuff, replace, and delete chunks.  Note that the
+ *  operations are done lazily--only if you convert the buffer to a
+ *  String.  This is very efficient because you are not moving data around
+ *  all the time.  As the buffer of tokens is converted to strings, the
+ *  toString() method(s) check to see if there is an operation at the
+ *  current index.  If so, the operation is done and then normal String
+ *  rendering continues on the buffer.  This is like having multiple Turing
+ *  machine instruction streams (programs) operating on a single input tape. :)
+ *
+ *  Since the operations are done lazily at toString-time, operations do not
+ *  screw up the token index values.  That is, an insert operation at token
+ *  index i does not change the index values for tokens i+1..n-1.
+ *
+ *  Because operations never actually alter the buffer, you may always get
+ *  the original token stream back without undoing anything.  Since
+ *  the instructions are queued up, you can easily simulate transactions and
+ *  roll back any changes if there is an error just by removing instructions.
+ *  For example,
+ *
+ *   CharStream input = new ANTLRFileStream("input");
+ *   TLexer lex = new TLexer(input);
+ *   TokenRewriteStream tokens = new TokenRewriteStream(lex);
+ *   T parser = new T(tokens);
+ *   parser.startRule();
+ *
+ * 	 Then in the rules, you can execute
+ *      Token t,u;
+ *      ...
+ *      input.insertAfter(t, "text to put after t");}
+ * 		input.insertAfter(u, "text after u");}
+ * 		System.out.println(tokens.toString());
+ *
+ *  Actually, you have to cast the 'input' to a TokenRewriteStream. :(
+ *
+ *  You can also have multiple "instruction streams" and get multiple
+ *  rewrites from a single pass over the input.  Just name the instruction
+ *  streams and use that name again when printing the buffer.  This could be
+ *  useful for generating a C file and also its header file--all from the
+ *  same buffer:
+ *
+ *      tokens.insertAfter("pass1", t, "text to put after t");}
+ * 		tokens.insertAfter("pass2", u, "text after u");}
+ * 		System.out.println(tokens.toString("pass1"));
+ * 		System.out.println(tokens.toString("pass2"));
+ *
+ *  If you don't use named rewrite streams, a "default" stream is used as
+ *  the first example shows.
+ */
+public class TokenRewriteStream extends CommonTokenStream {
+	public static final String DEFAULT_PROGRAM_NAME = "default";
+    public static final int PROGRAM_INIT_SIZE = 100;
+	public static final int MIN_TOKEN_INDEX = 0;
+
+	// Define the rewrite operation hierarchy
+
+	class RewriteOperation {
+        /** What index into rewrites List are we? */
+        protected int instructionIndex;
+        /** Token buffer index. */
+        protected int index;
+		protected Object text;
+
+		protected RewriteOperation(int index) {
+			this.index = index;
+		}
+
+		protected RewriteOperation(int index, Object text) {
+			this.index = index;
+			this.text = text;
+		}
+		/** Execute the rewrite operation by possibly adding to the buffer.
+		 *  Return the index of the next token to operate on.
+		 */
+		public int execute(StringBuffer buf) {
+			return index;
+		}
+		public String toString() {
+			String opName = getClass().getName();
+			int $index = opName.indexOf('$');
+			opName = opName.substring($index+1, opName.length());
+			return "<"+opName+"@"+tokens.get(index)+
+				   ":\""+text+"\">";
+		}
+	}
+
+	class InsertBeforeOp extends RewriteOperation {
+		public InsertBeforeOp(int index, Object text) {
+			super(index,text);
+		}
+		public int execute(StringBuffer buf) {
+			buf.append(text);
+			if ( tokens.get(index).getType()!=Token.EOF ) {
+				buf.append(tokens.get(index).getText());
+			}
+			return index+1;
+		}
+	}
+
+	/** I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp
+	 *  instructions.
+	 */
+	class ReplaceOp extends RewriteOperation {
+		protected int lastIndex;
+		public ReplaceOp(int from, int to, Object text) {
+			super(from,text);
+			lastIndex = to;
+		}
+		public int execute(StringBuffer buf) {
+			if ( text!=null ) {
+				buf.append(text);
+			}
+			return lastIndex+1;
+		}
+		public String toString() {
+			if ( text==null ) {
+				return "<DeleteOp@"+tokens.get(index)+
+					   ".."+tokens.get(lastIndex)+">";
+			}
+			return "<ReplaceOp@"+tokens.get(index)+
+				   ".."+tokens.get(lastIndex)+":\""+text+"\">";
+		}
+	}
+
+	/** You may have multiple, named streams of rewrite operations.
+	 *  I'm calling these things "programs."
+	 *  Maps String (name) -> rewrite (List)
+	 */
+	protected Map programs = null;
+
+	/** Map String (program name) -> Integer index */
+	protected Map lastRewriteTokenIndexes = null;
+
+	public TokenRewriteStream() {
+		init();
+	}
+
+	protected void init() {
+		programs = new HashMap();
+		programs.put(DEFAULT_PROGRAM_NAME, new ArrayList(PROGRAM_INIT_SIZE));
+		lastRewriteTokenIndexes = new HashMap();
+	}
+
+	public TokenRewriteStream(TokenSource tokenSource) {
+	    super(tokenSource);
+		init();
+	}
+
+	public TokenRewriteStream(TokenSource tokenSource, int channel) {
+		super(tokenSource, channel);
+		init();
+	}
+
+	public void rollback(int instructionIndex) {
+		rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
+	}
+
+	/** Rollback the instruction stream for a program so that
+	 *  the indicated instruction (via instructionIndex) is no
+	 *  longer in the stream.  UNTESTED!
+	 */
+	public void rollback(String programName, int instructionIndex) {
+		List is = (List)programs.get(programName);
+		if ( is!=null ) {
+			programs.put(programName, is.subList(MIN_TOKEN_INDEX,instructionIndex));
+		}
+	}
+
+	public void deleteProgram() {
+		deleteProgram(DEFAULT_PROGRAM_NAME);
+	}
+
+	/** Reset the program so that no instructions exist */
+	public void deleteProgram(String programName) {
+		rollback(programName, MIN_TOKEN_INDEX);
+	}
+
+	public void insertAfter(Token t, Object text) {
+		insertAfter(DEFAULT_PROGRAM_NAME, t, text);
+	}
+
+	public void insertAfter(int index, Object text) {
+		insertAfter(DEFAULT_PROGRAM_NAME, index, text);
+	}
+
+	public void insertAfter(String programName, Token t, Object text) {
+		insertAfter(programName,t.getTokenIndex(), text);
+	}
+
+	public void insertAfter(String programName, int index, Object text) {
+		// to insert after, just insert before next index (even if past end)
+		insertBefore(programName,index+1, text);
+	}
+
+	public void insertBefore(Token t, Object text) {
+		insertBefore(DEFAULT_PROGRAM_NAME, t, text);
+	}
+
+	public void insertBefore(int index, Object text) {
+		insertBefore(DEFAULT_PROGRAM_NAME, index, text);
+	}
+
+	public void insertBefore(String programName, Token t, Object text) {
+		insertBefore(programName, t.getTokenIndex(), text);
+	}
+
+	public void insertBefore(String programName, int index, Object text) {
+		RewriteOperation op = new InsertBeforeOp(index,text);
+		List rewrites = getProgram(programName);
+        op.instructionIndex = rewrites.size();
+        rewrites.add(op);		
+	}
+
+	public void replace(int index, Object text) {
+		replace(DEFAULT_PROGRAM_NAME, index, index, text);
+	}
+
+	public void replace(int from, int to, Object text) {
+		replace(DEFAULT_PROGRAM_NAME, from, to, text);
+	}
+
+	public void replace(Token indexT, Object text) {
+		replace(DEFAULT_PROGRAM_NAME, indexT, indexT, text);
+	}
+
+	public void replace(Token from, Token to, Object text) {
+		replace(DEFAULT_PROGRAM_NAME, from, to, text);
+	}
+
+	public void replace(String programName, int from, int to, Object text) {
+		if ( from > to || from<0 || to<0 || to >= tokens.size() ) {
+			throw new IllegalArgumentException("replace: range invalid: "+from+".."+to+"(size="+tokens.size()+")");
+		}
+		RewriteOperation op = new ReplaceOp(from, to, text);
+		List rewrites = getProgram(programName);
+        op.instructionIndex = rewrites.size();
+        rewrites.add(op);
+	}
+
+	public void replace(String programName, Token from, Token to, Object text) {
+		replace(programName,
+				from.getTokenIndex(),
+				to.getTokenIndex(),
+				text);
+	}
+
+	public void delete(int index) {
+		delete(DEFAULT_PROGRAM_NAME, index, index);
+	}
+
+	public void delete(int from, int to) {
+		delete(DEFAULT_PROGRAM_NAME, from, to);
+	}
+
+	public void delete(Token indexT) {
+		delete(DEFAULT_PROGRAM_NAME, indexT, indexT);
+	}
+
+	public void delete(Token from, Token to) {
+		delete(DEFAULT_PROGRAM_NAME, from, to);
+	}
+
+	public void delete(String programName, int from, int to) {
+		replace(programName,from,to,null);
+	}
+
+	public void delete(String programName, Token from, Token to) {
+		replace(programName,from,to,null);
+	}
+
+	public int getLastRewriteTokenIndex() {
+		return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
+	}
+
+	protected int getLastRewriteTokenIndex(String programName) {
+		Integer I = (Integer)lastRewriteTokenIndexes.get(programName);
+		if ( I==null ) {
+			return -1;
+		}
+		return I.intValue();
+	}
+
+	protected void setLastRewriteTokenIndex(String programName, int i) {
+		lastRewriteTokenIndexes.put(programName, new Integer(i));
+	}
+
+	protected List getProgram(String name) {
+		List is = (List)programs.get(name);
+		if ( is==null ) {
+			is = initializeProgram(name);
+		}
+		return is;
+	}
+
+	private List initializeProgram(String name) {
+		List is = new ArrayList(PROGRAM_INIT_SIZE);
+		programs.put(name, is);
+		return is;
+	}
+
+	public String toOriginalString() {
+        fill();
+		return toOriginalString(MIN_TOKEN_INDEX, size()-1);
+	}
+
+	public String toOriginalString(int start, int end) {
+		StringBuffer buf = new StringBuffer();
+		for (int i=start; i>=MIN_TOKEN_INDEX && i<=end && i<tokens.size(); i++) {
+			if ( get(i).getType()!=Token.EOF ) buf.append(get(i).getText());
+		}
+		return buf.toString();
+	}
+
+	public String toString() {
+        fill();
+		return toString(MIN_TOKEN_INDEX, size()-1);
+	}
+
+	public String toString(String programName) {
+        fill();
+		return toString(programName, MIN_TOKEN_INDEX, size()-1);
+	}
+
+	public String toString(int start, int end) {
+		return toString(DEFAULT_PROGRAM_NAME, start, end);
+	}
+
+	public String toString(String programName, int start, int end) {
+		List rewrites = (List)programs.get(programName);
+
+        // ensure start/end are in range
+        if ( end>tokens.size()-1 ) end = tokens.size()-1;
+        if ( start<0 ) start = 0;
+
+        if ( rewrites==null || rewrites.size()==0 ) {
+			return toOriginalString(start,end); // no instructions to execute
+		}
+		StringBuffer buf = new StringBuffer();
+
+		// First, optimize instruction stream
+		Map indexToOp = reduceToSingleOperationPerIndex(rewrites);
+
+        // Walk buffer, executing instructions and emitting tokens
+        int i = start;
+        while ( i <= end && i < tokens.size() ) {
+			RewriteOperation op = (RewriteOperation)indexToOp.get(new Integer(i));
+			indexToOp.remove(new Integer(i)); // remove so any left have index size-1
+			Token t = (Token) tokens.get(i);
+			if ( op==null ) {
+				// no operation at that index, just dump token
+				if ( t.getType()!=Token.EOF ) buf.append(t.getText());
+				i++; // move to next token
+			}
+			else {
+				i = op.execute(buf); // execute operation and skip
+			}
+		}
+
+        // include stuff after end if it's last index in buffer
+        // So, if they did an insertAfter(lastValidIndex, "foo"), include
+        // foo if end==lastValidIndex.
+        if ( end==tokens.size()-1 ) {
+            // Scan any remaining operations after last token
+            // should be included (they will be inserts).
+            Iterator it = indexToOp.values().iterator();
+            while (it.hasNext()) {
+                RewriteOperation op = (RewriteOperation)it.next();
+                if ( op.index >= tokens.size()-1 ) buf.append(op.text);
+            }
+        }
+        return buf.toString();
+	}
+
+	/** We need to combine operations and report invalid operations (like
+	 *  overlapping replaces that are not completed nested).  Inserts to
+	 *  same index need to be combined etc...   Here are the cases:
+	 *
+	 *  I.i.u I.j.v								leave alone, nonoverlapping
+	 *  I.i.u I.i.v								combine: Iivu
+	 *
+	 *  R.i-j.u R.x-y.v	| i-j in x-y			delete first R
+	 *  R.i-j.u R.i-j.v							delete first R
+	 *  R.i-j.u R.x-y.v	| x-y in i-j			ERROR
+	 *  R.i-j.u R.x-y.v	| boundaries overlap	ERROR
+	 *
+	 *  Delete special case of replace (text==null):
+	 *  D.i-j.u D.x-y.v	| boundaries overlap	combine to max(min)..max(right)
+	 *
+	 *  I.i.u R.x-y.v | i in (x+1)-y			delete I (since insert before
+	 *											we're not deleting i)
+	 *  I.i.u R.x-y.v | i not in (x+1)-y		leave alone, nonoverlapping
+	 *  R.x-y.v I.i.u | i in x-y				ERROR
+	 *  R.x-y.v I.x.u 							R.x-y.uv (combine, delete I)
+	 *  R.x-y.v I.i.u | i not in x-y			leave alone, nonoverlapping
+	 *
+	 *  I.i.u = insert u before op @ index i
+	 *  R.x-y.u = replace x-y indexed tokens with u
+	 *
+	 *  First we need to examine replaces.  For any replace op:
+	 *
+	 * 		1. wipe out any insertions before op within that range.
+	 *		2. Drop any replace op before that is contained completely within
+	 *         that range.
+	 *		3. Throw exception upon boundary overlap with any previous replace.
+	 *
+	 *  Then we can deal with inserts:
+	 *
+	 * 		1. for any inserts to same index, combine even if not adjacent.
+	 * 		2. for any prior replace with same left boundary, combine this
+	 *         insert with replace and delete this replace.
+	 * 		3. throw exception if index in same range as previous replace
+	 *
+	 *  Don't actually delete; make op null in list. Easier to walk list.
+	 *  Later we can throw as we add to index -> op map.
+	 *
+	 *  Note that I.2 R.2-2 will wipe out I.2 even though, technically, the
+	 *  inserted stuff would be before the replace range.  But, if you
+	 *  add tokens in front of a method body '{' and then delete the method
+	 *  body, I think the stuff before the '{' you added should disappear too.
+	 *
+	 *  Return a map from token index to operation.
+	 */
+	protected Map reduceToSingleOperationPerIndex(List rewrites) {
+//		System.out.println("rewrites="+rewrites);
+
+		// WALK REPLACES
+		for (int i = 0; i < rewrites.size(); i++) {
+			RewriteOperation op = (RewriteOperation)rewrites.get(i);
+			if ( op==null ) continue;
+			if ( !(op instanceof ReplaceOp) ) continue;
+			ReplaceOp rop = (ReplaceOp)rewrites.get(i);
+			// Wipe prior inserts within range
+			List inserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
+			for (int j = 0; j < inserts.size(); j++) {
+				InsertBeforeOp iop = (InsertBeforeOp) inserts.get(j);
+				if ( iop.index == rop.index ) {
+					// E.g., insert before 2, delete 2..2; update replace
+					// text to include insert before, kill insert
+					rewrites.set(iop.instructionIndex, null);
+					rop.text = iop.text.toString() + (rop.text!=null?rop.text.toString():"");
+				}
+				else if ( iop.index > rop.index && iop.index <= rop.lastIndex ) {
+                    // delete insert as it's a no-op.
+                    rewrites.set(iop.instructionIndex, null);
+				}
+			}
+			// Drop any prior replaces contained within
+			List prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i);
+			for (int j = 0; j < prevReplaces.size(); j++) {
+				ReplaceOp prevRop = (ReplaceOp) prevReplaces.get(j);
+				if ( prevRop.index>=rop.index && prevRop.lastIndex <= rop.lastIndex ) {
+                    // delete replace as it's a no-op.
+                    rewrites.set(prevRop.instructionIndex, null);
+					continue;
+				}
+				// throw exception unless disjoint or identical
+				boolean disjoint =
+					prevRop.lastIndex<rop.index || prevRop.index > rop.lastIndex;
+				boolean same =
+					prevRop.index==rop.index && prevRop.lastIndex==rop.lastIndex;
+				// Delete special case of replace (text==null):
+				// D.i-j.u D.x-y.v	| boundaries overlap	combine to max(min)..max(right)
+				if ( prevRop.text==null && rop.text==null && !disjoint ) {
+					//System.out.println("overlapping deletes: "+prevRop+", "+rop);
+					rewrites.set(prevRop.instructionIndex, null); // kill first delete
+					rop.index = Math.min(prevRop.index, rop.index);
+					rop.lastIndex = Math.max(prevRop.lastIndex, rop.lastIndex);
+					System.out.println("new rop "+rop);
+				}
+				else if ( !disjoint && !same ) {
+					throw new IllegalArgumentException("replace op boundaries of "+rop+
+													   " overlap with previous "+prevRop);
+				}
+			}
+		}
+
+		// WALK INSERTS
+		for (int i = 0; i < rewrites.size(); i++) {
+			RewriteOperation op = (RewriteOperation)rewrites.get(i);
+			if ( op==null ) continue;
+			if ( !(op instanceof InsertBeforeOp) ) continue;
+			InsertBeforeOp iop = (InsertBeforeOp)rewrites.get(i);
+			// combine current insert with prior if any at same index
+			List prevInserts = getKindOfOps(rewrites, InsertBeforeOp.class, i);
+			for (int j = 0; j < prevInserts.size(); j++) {
+				InsertBeforeOp prevIop = (InsertBeforeOp) prevInserts.get(j);
+				if ( prevIop.index == iop.index ) { // combine objects
+					// convert to strings...we're in process of toString'ing
+					// whole token buffer so no lazy eval issue with any templates
+					iop.text = catOpText(iop.text,prevIop.text);
+                    // delete redundant prior insert
+                    rewrites.set(prevIop.instructionIndex, null);
+				}
+			}
+			// look for replaces where iop.index is in range; error
+			List prevReplaces = getKindOfOps(rewrites, ReplaceOp.class, i);
+			for (int j = 0; j < prevReplaces.size(); j++) {
+				ReplaceOp rop = (ReplaceOp) prevReplaces.get(j);
+				if ( iop.index == rop.index ) {
+					rop.text = catOpText(iop.text,rop.text);
+					rewrites.set(i, null);  // delete current insert
+					continue;
+				}
+				if ( iop.index >= rop.index && iop.index <= rop.lastIndex ) {
+					throw new IllegalArgumentException("insert op "+iop+
+													   " within boundaries of previous "+rop);
+				}
+			}
+		}
+		// System.out.println("rewrites after="+rewrites);
+		Map m = new HashMap();
+		for (int i = 0; i < rewrites.size(); i++) {
+			RewriteOperation op = (RewriteOperation)rewrites.get(i);
+			if ( op==null ) continue; // ignore deleted ops
+			if ( m.get(new Integer(op.index))!=null ) {
+				throw new Error("should only be one op per index");
+			}
+			m.put(new Integer(op.index), op);
+		}
+		//System.out.println("index to op: "+m);
+		return m;
+	}
+
+	protected String catOpText(Object a, Object b) {
+		String x = "";
+		String y = "";
+		if ( a!=null ) x = a.toString();
+		if ( b!=null ) y = b.toString();
+		return x+y;
+	}
+	protected List getKindOfOps(List rewrites, Class kind) {
+		return getKindOfOps(rewrites, kind, rewrites.size());
+	}
+
+    /** Get all operations before an index of a particular kind */
+    protected List getKindOfOps(List rewrites, Class kind, int before) {
+		List ops = new ArrayList();
+		for (int i=0; i<before && i<rewrites.size(); i++) {
+			RewriteOperation op = (RewriteOperation)rewrites.get(i);
+			if ( op==null ) continue; // ignore deleted
+			if ( op.getClass() == kind ) ops.add(op);
+		}		
+		return ops;
+	}
+
+	public String toDebugString() {
+		return toDebugString(MIN_TOKEN_INDEX, size()-1);
+	}
+
+	public String toDebugString(int start, int end) {
+		StringBuffer buf = new StringBuffer();
+		for (int i=start; i>=MIN_TOKEN_INDEX && i<=end && i<tokens.size(); i++) {
+			buf.append(get(i));
+		}
+		return buf.toString();
+	}
+}
diff --git a/src/org/antlr/runtime/TokenSource.java b/src/org/antlr/runtime/TokenSource.java
new file mode 100755
index 0000000..1f4dc64
--- /dev/null
+++ b/src/org/antlr/runtime/TokenSource.java
@@ -0,0 +1,54 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** A source of tokens must provide a sequence of tokens via nextToken()
+ *  and also must reveal it's source of characters; CommonToken's text is
+ *  computed from a CharStream; it only store indices into the char stream.
+ *
+ *  Errors from the lexer are never passed to the parser.  Either you want
+ *  to keep going or you do not upon token recognition error.  If you do not
+ *  want to continue lexing then you do not want to continue parsing.  Just
+ *  throw an exception not under RecognitionException and Java will naturally
+ *  toss you all the way out of the recognizers.  If you want to continue
+ *  lexing then you should not throw an exception to the parser--it has already
+ *  requested a token.  Keep lexing until you get a valid one.  Just report
+ *  errors and keep going, looking for a valid token.
+ */
+public interface TokenSource {
+	/** Return a Token object from your input stream (usually a CharStream).
+	 *  Do not fail/return upon lexing error; keep chewing on the characters
+	 *  until you get a good one; errors are not passed through to the parser.
+	 */
+	public Token nextToken();
+
+	/** Where are you getting tokens from? normally the implication will simply
+	 *  ask lexers input stream.
+	 */
+	public String getSourceName();
+}
diff --git a/src/org/antlr/runtime/TokenStream.java b/src/org/antlr/runtime/TokenStream.java
new file mode 100755
index 0000000..1b43c14
--- /dev/null
+++ b/src/org/antlr/runtime/TokenStream.java
@@ -0,0 +1,75 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import java.util.List;
+
+/** A stream of tokens accessing tokens from a TokenSource */
+public interface TokenStream extends IntStream {
+    /** Get Token at current input pointer + i ahead where i=1 is next Token.
+	 *  i<0 indicates tokens in the past.  So -1 is previous token and -2 is
+	 *  two tokens ago. LT(0) is undefined.  For i>=n, return Token.EOFToken.
+	 *  Return null for LT(0) and any index that results in an absolute address
+	 *  that is negative.
+	 */
+    public Token LT(int k);
+
+	/** How far ahead has the stream been asked to look?  The return
+	 *  value is a valid index from 0..n-1.
+	 */
+	int range();
+	
+	/** Get a token at an absolute index i; 0..n-1.  This is really only
+	 *  needed for profiling and debugging and token stream rewriting.
+	 *  If you don't want to buffer up tokens, then this method makes no
+	 *  sense for you.  Naturally you can't use the rewrite stream feature.
+	 *  I believe DebugTokenStream can easily be altered to not use
+	 *  this method, removing the dependency.
+	 */
+	public Token get(int i);
+
+	/** Where is this stream pulling tokens from?  This is not the name, but
+	 *  the object that provides Token objects.
+	 */
+	public TokenSource getTokenSource();
+
+	/** Return the text of all tokens from start to stop, inclusive.
+	 *  If the stream does not buffer all the tokens then it can just
+	 *  return "" or null;  Users should not access $ruleLabel.text in
+	 *  an action of course in that case.
+	 */
+	public String toString(int start, int stop);
+
+	/** Because the user is not required to use a token with an index stored
+	 *  in it, we must provide a means for two token objects themselves to
+	 *  indicate the start/end location.  Most often this will just delegate
+	 *  to the other toString(int,int).  This is also parallel with
+	 *  the TreeNodeStream.toString(Object,Object).
+	 */
+	public String toString(Token start, Token stop);
+}
diff --git a/src/org/antlr/runtime/UnbufferedTokenStream.java b/src/org/antlr/runtime/UnbufferedTokenStream.java
new file mode 100755
index 0000000..0b0e979
--- /dev/null
+++ b/src/org/antlr/runtime/UnbufferedTokenStream.java
@@ -0,0 +1,82 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+import org.antlr.runtime.misc.LookaheadStream;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/** A token stream that pulls tokens from the code source on-demand and
+ *  without tracking a complete buffer of the tokens. This stream buffers
+ *  the minimum number of tokens possible.  It's the same as
+ *  OnDemandTokenStream except that OnDemandTokenStream buffers all tokens.
+ *
+ *  You can't use this stream if you pass whitespace or other off-channel
+ *  tokens to the parser. The stream can't ignore off-channel tokens.
+ * 
+ *  You can only look backwards 1 token: LT(-1).
+ *
+ *  Use this when you need to read from a socket or other infinite stream.
+ *
+ *  @see BufferedTokenStream
+ *  @see CommonTokenStream
+ */
+public class UnbufferedTokenStream extends LookaheadStream<Token> implements TokenStream {
+	protected TokenSource tokenSource;
+    protected int tokenIndex = 0; // simple counter to set token index in tokens
+
+    /** Skip tokens on any channel but this one; this is how we skip whitespace... */
+    protected int channel = Token.DEFAULT_CHANNEL;
+
+	public UnbufferedTokenStream(TokenSource tokenSource) {
+		this.tokenSource = tokenSource;
+	}
+
+	public Token nextElement() {
+		Token t = tokenSource.nextToken();
+        t.setTokenIndex(tokenIndex++);
+		return t;
+	}
+
+    public boolean isEOF(Token o) { return o.getType() == Token.EOF; }    
+
+	public TokenSource getTokenSource() { return tokenSource; }
+
+	public String toString(int start, int stop) { return "n/a"; }
+
+	public String toString(Token start, Token stop) { return "n/a"; }
+
+    public int LA(int i) { return LT(i).getType(); }
+
+    public Token get(int i) {
+        throw new UnsupportedOperationException("Absolute token indexes are meaningless in an unbuffered stream");
+    }
+
+	public String getSourceName() {	return tokenSource.getSourceName();	}
+}
diff --git a/src/org/antlr/runtime/UnwantedTokenException.java b/src/org/antlr/runtime/UnwantedTokenException.java
new file mode 100755
index 0000000..feb7445
--- /dev/null
+++ b/src/org/antlr/runtime/UnwantedTokenException.java
@@ -0,0 +1,53 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime;
+
+/** An extra token while parsing a TokenStream */
+public class UnwantedTokenException extends MismatchedTokenException {
+	/** Used for remote debugger deserialization */
+	public UnwantedTokenException() {;}
+
+	public UnwantedTokenException(int expecting, IntStream input) {
+		super(expecting, input);
+	}
+
+	public Token getUnexpectedToken() {
+		return token;
+	}
+
+	public String toString() {
+		String exp = ", expected "+expecting;
+		if ( expecting==Token.INVALID_TOKEN_TYPE ) {
+			exp = "";
+		}
+		if ( token==null ) {
+			return "UnwantedTokenException(found="+null+exp+")";
+		}
+		return "UnwantedTokenException(found="+token.getText()+exp+")";
+	}
+}
diff --git a/src/org/antlr/runtime/debug/BlankDebugEventListener.java b/src/org/antlr/runtime/debug/BlankDebugEventListener.java
new file mode 100755
index 0000000..d70aa26
--- /dev/null
+++ b/src/org/antlr/runtime/debug/BlankDebugEventListener.java
@@ -0,0 +1,77 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+
+/** A blank listener that does nothing; useful for real classes so
+ *  they don't have to have lots of blank methods and are less
+ *  sensitive to updates to debug interface.
+ */
+public class BlankDebugEventListener implements DebugEventListener {
+	public void enterRule(String grammarFileName, String ruleName) {}
+	public void exitRule(String grammarFileName, String ruleName) {}
+	public void enterAlt(int alt) {}
+	public void enterSubRule(int decisionNumber) {}
+	public void exitSubRule(int decisionNumber) {}
+	public void enterDecision(int decisionNumber, boolean couldBacktrack) {}
+	public void exitDecision(int decisionNumber) {}
+	public void location(int line, int pos) {}
+	public void consumeToken(Token token) {}
+	public void consumeHiddenToken(Token token) {}
+	public void LT(int i, Token t) {}
+	public void mark(int i) {}
+	public void rewind(int i) {}
+	public void rewind() {}
+	public void beginBacktrack(int level) {}
+	public void endBacktrack(int level, boolean successful) {}
+	public void recognitionException(RecognitionException e) {}
+	public void beginResync() {}
+	public void endResync() {}
+	public void semanticPredicate(boolean result, String predicate) {}
+	public void commence() {}
+	public void terminate() {}
+
+	// Tree parsing stuff
+
+	public void consumeNode(Object t) {}
+	public void LT(int i, Object t) {}
+
+	// AST Stuff
+
+	public void nilNode(Object t) {}
+	public void errorNode(Object t) {}
+	public void createNode(Object t) {}
+	public void createNode(Object node, Token token) {}
+	public void becomeRoot(Object newRoot, Object oldRoot) {}
+	public void addChild(Object root, Object child) {}
+	public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex) {}
+}
+
+
diff --git a/src/org/antlr/runtime/debug/DebugEventHub.java b/src/org/antlr/runtime/debug/DebugEventHub.java
new file mode 100755
index 0000000..7bfe6a8
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugEventHub.java
@@ -0,0 +1,292 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.RecognitionException;
+
+import java.util.List;
+import java.util.ArrayList;
+
+/** Broadcast debug events to multiple listeners.  Lets you debug and still
+ *  use the event mechanism to build parse trees etc...  Not thread-safe.
+ *  Don't add events in one thread while parser fires events in another.
+ * 
+ *  @see also DebugEventRepeater
+ */
+public class DebugEventHub implements DebugEventListener {
+	protected List listeners = new ArrayList();
+
+	public DebugEventHub(DebugEventListener listener) {
+		listeners.add(listener);
+	}
+
+	public DebugEventHub(DebugEventListener a, DebugEventListener b) {
+		listeners.add(a);
+		listeners.add(b);
+	}
+
+	/** Add another listener to broadcast events too.  Not thread-safe.
+	 *  Don't add events in one thread while parser fires events in another.
+	 */
+	public void addListener(DebugEventListener listener) {
+		listeners.add(listener);
+	}
+	
+	/* To avoid a mess like this:
+		public void enterRule(final String ruleName) {
+			broadcast(new Code(){
+				public void exec(DebugEventListener listener) {listener.enterRule(ruleName);}}
+				);
+		}
+		I am dup'ing the for-loop in each.  Where are Java closures!? blech!
+	 */
+
+	public void enterRule(String grammarFileName, String ruleName) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.enterRule(grammarFileName,ruleName);
+		}
+	}
+
+	public void exitRule(String grammarFileName, String ruleName) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.exitRule(grammarFileName, ruleName);
+		}
+	}
+
+	public void enterAlt(int alt) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.enterAlt(alt);
+		}
+	}
+
+	public void enterSubRule(int decisionNumber) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.enterSubRule(decisionNumber);
+		}
+	}
+
+	public void exitSubRule(int decisionNumber) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.exitSubRule(decisionNumber);
+		}
+	}
+
+	public void enterDecision(int decisionNumber, boolean couldBacktrack) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.enterDecision(decisionNumber, couldBacktrack);
+		}
+	}
+
+	public void exitDecision(int decisionNumber) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.exitDecision(decisionNumber);
+		}
+	}
+
+	public void location(int line, int pos) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.location(line, pos);
+		}
+	}
+
+	public void consumeToken(Token token) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.consumeToken(token);
+		}
+	}
+
+	public void consumeHiddenToken(Token token) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.consumeHiddenToken(token);
+		}
+	}
+
+	public void LT(int index, Token t) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.LT(index, t);
+		}
+	}
+
+	public void mark(int index) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.mark(index);
+		}
+	}
+
+	public void rewind(int index) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.rewind(index);
+		}
+	}
+
+	public void rewind() {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.rewind();
+		}
+	}
+
+	public void beginBacktrack(int level) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.beginBacktrack(level);
+		}
+	}
+
+	public void endBacktrack(int level, boolean successful) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.endBacktrack(level, successful);
+		}
+	}
+
+	public void recognitionException(RecognitionException e) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.recognitionException(e);
+		}
+	}
+
+	public void beginResync() {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.beginResync();
+		}
+	}
+
+	public void endResync() {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.endResync();
+		}
+	}
+
+	public void semanticPredicate(boolean result, String predicate) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.semanticPredicate(result, predicate);
+		}
+	}
+
+	public void commence() {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.commence();
+		}
+	}
+
+	public void terminate() {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.terminate();
+		}
+	}
+
+
+	// Tree parsing stuff
+
+	public void consumeNode(Object t) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.consumeNode(t);
+		}
+	}
+
+	public void LT(int index, Object t) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.LT(index, t);
+		}
+	}
+
+
+	// AST Stuff
+
+	public void nilNode(Object t) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.nilNode(t);
+		}
+	}
+
+	public void errorNode(Object t) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.errorNode(t);
+		}
+	}
+
+	public void createNode(Object t) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.createNode(t);
+		}
+	}
+
+	public void createNode(Object node, Token token) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.createNode(node, token);
+		}
+	}
+
+	public void becomeRoot(Object newRoot, Object oldRoot) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.becomeRoot(newRoot, oldRoot);
+		}
+	}
+
+	public void addChild(Object root, Object child) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.addChild(root, child);
+		}
+	}
+
+	public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex) {
+		for (int i = 0; i < listeners.size(); i++) {
+			DebugEventListener listener = (DebugEventListener)listeners.get(i);
+			listener.setTokenBoundaries(t, tokenStartIndex, tokenStopIndex);
+		}
+	}
+}
diff --git a/src/org/antlr/runtime/debug/DebugEventListener.java b/src/org/antlr/runtime/debug/DebugEventListener.java
new file mode 100755
index 0000000..163b5cd
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugEventListener.java
@@ -0,0 +1,323 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+
+/** All debugging events that a recognizer can trigger.
+ *
+ *  I did not create a separate AST debugging interface as it would create
+ *  lots of extra classes and DebugParser has a dbg var defined, which makes
+ *  it hard to change to ASTDebugEventListener.  I looked hard at this issue
+ *  and it is easier to understand as one monolithic event interface for all
+ *  possible events.  Hopefully, adding ST debugging stuff won't be bad.  Leave
+ *  for future. 4/26/2006.
+ */
+public interface DebugEventListener {
+	/** Moved to version 2 for v3.1: added grammar name to enter/exit Rule */
+	public static final String PROTOCOL_VERSION = "2";
+	
+	/** serialized version of true */
+	public static final int TRUE = 1;
+	public static final int FALSE = 0;
+
+	/** The parser has just entered a rule.  No decision has been made about
+	 *  which alt is predicted.  This is fired AFTER init actions have been
+	 *  executed.  Attributes are defined and available etc...
+	 *  The grammarFileName allows composite grammars to jump around among
+	 *  multiple grammar files.
+	 */
+	public void enterRule(String grammarFileName, String ruleName);
+
+	/** Because rules can have lots of alternatives, it is very useful to
+	 *  know which alt you are entering.  This is 1..n for n alts.
+	 */
+	public void enterAlt(int alt);
+
+	/** This is the last thing executed before leaving a rule.  It is
+	 *  executed even if an exception is thrown.  This is triggered after
+	 *  error reporting and recovery have occurred (unless the exception is
+	 *  not caught in this rule).  This implies an "exitAlt" event.
+	 *  The grammarFileName allows composite grammars to jump around among
+	 *  multiple grammar files.
+	 */
+	public void exitRule(String grammarFileName, String ruleName);
+
+	/** Track entry into any (...) subrule other EBNF construct */
+	public void enterSubRule(int decisionNumber);
+
+	public void exitSubRule(int decisionNumber);
+
+	/** Every decision, fixed k or arbitrary, has an enter/exit event
+	 *  so that a GUI can easily track what LT/consume events are
+	 *  associated with prediction.  You will see a single enter/exit
+	 *  subrule but multiple enter/exit decision events, one for each
+	 *  loop iteration.
+	 */
+	public void enterDecision(int decisionNumber, boolean couldBacktrack);
+
+	public void exitDecision(int decisionNumber);
+
+	/** An input token was consumed; matched by any kind of element.
+	 *  Trigger after the token was matched by things like match(), matchAny().
+	 */
+	public void consumeToken(Token t);
+
+	/** An off-channel input token was consumed.
+	 *  Trigger after the token was matched by things like match(), matchAny().
+	 *  (unless of course the hidden token is first stuff in the input stream).
+	 */
+	public void consumeHiddenToken(Token t);
+
+	/** Somebody (anybody) looked ahead.  Note that this actually gets
+	 *  triggered by both LA and LT calls.  The debugger will want to know
+	 *  which Token object was examined.  Like consumeToken, this indicates
+	 *  what token was seen at that depth.  A remote debugger cannot look
+	 *  ahead into a file it doesn't have so LT events must pass the token
+	 *  even if the info is redundant.
+	 */
+	public void LT(int i, Token t);
+
+	/** The parser is going to look arbitrarily ahead; mark this location,
+	 *  the token stream's marker is sent in case you need it.
+	 */
+	public void mark(int marker);
+
+	/** After an arbitrairly long lookahead as with a cyclic DFA (or with
+	 *  any backtrack), this informs the debugger that stream should be
+	 *  rewound to the position associated with marker.
+	 */
+	public void rewind(int marker);
+
+	/** Rewind to the input position of the last marker.
+	 *  Used currently only after a cyclic DFA and just
+	 *  before starting a sem/syn predicate to get the
+	 *  input position back to the start of the decision.
+	 *  Do not "pop" the marker off the state.  mark(i)
+	 *  and rewind(i) should balance still.
+	 */
+	public void rewind();
+
+	public void beginBacktrack(int level);
+
+	public void endBacktrack(int level, boolean successful);
+
+	/** To watch a parser move through the grammar, the parser needs to
+	 *  inform the debugger what line/charPos it is passing in the grammar.
+	 *  For now, this does not know how to switch from one grammar to the
+	 *  other and back for island grammars etc...
+	 *
+	 *  This should also allow breakpoints because the debugger can stop
+	 *  the parser whenever it hits this line/pos.
+	 */
+	public void location(int line, int pos);
+
+	/** A recognition exception occurred such as NoViableAltException.  I made
+	 *  this a generic event so that I can alter the exception hierachy later
+	 *  without having to alter all the debug objects.
+	 *
+	 *  Upon error, the stack of enter rule/subrule must be properly unwound.
+	 *  If no viable alt occurs it is within an enter/exit decision, which
+	 *  also must be rewound.  Even the rewind for each mark must be unwount.
+	 *  In the Java target this is pretty easy using try/finally, if a bit
+	 *  ugly in the generated code.  The rewind is generated in DFA.predict()
+	 *  actually so no code needs to be generated for that.  For languages
+	 *  w/o this "finally" feature (C++?), the target implementor will have
+	 *  to build an event stack or something.
+	 *
+	 *  Across a socket for remote debugging, only the RecognitionException
+	 *  data fields are transmitted.  The token object or whatever that
+	 *  caused the problem was the last object referenced by LT.  The
+	 *  immediately preceding LT event should hold the unexpected Token or
+	 *  char.
+	 *
+	 *  Here is a sample event trace for grammar:
+	 *
+	 *  b : C ({;}A|B) // {;} is there to prevent A|B becoming a set
+     *    | D
+     *    ;
+     *
+	 *  The sequence for this rule (with no viable alt in the subrule) for
+	 *  input 'c c' (there are 3 tokens) is:
+	 *
+	 *		commence
+	 *		LT(1)
+	 *		enterRule b
+	 *		location 7 1
+	 *		enter decision 3
+	 *		LT(1)
+	 *		exit decision 3
+	 *		enterAlt1
+	 *		location 7 5
+	 *		LT(1)
+	 *		consumeToken [c/<4>,1:0]
+	 *		location 7 7
+	 *		enterSubRule 2
+	 *		enter decision 2
+	 *		LT(1)
+	 *		LT(1)
+	 *		recognitionException NoViableAltException 2 1 2
+	 *		exit decision 2
+	 *		exitSubRule 2
+	 *		beginResync
+	 *		LT(1)
+	 *		consumeToken [c/<4>,1:1]
+	 *		LT(1)
+	 *		endResync
+	 *		LT(-1)
+	 *		exitRule b
+	 *		terminate
+	 */
+	public void recognitionException(RecognitionException e);
+
+	/** Indicates the recognizer is about to consume tokens to resynchronize
+	 *  the parser.  Any consume events from here until the recovered event
+	 *  are not part of the parse--they are dead tokens.
+	 */
+	public void beginResync();
+
+	/** Indicates that the recognizer has finished consuming tokens in order
+	 *  to resychronize.  There may be multiple beginResync/endResync pairs
+	 *  before the recognizer comes out of errorRecovery mode (in which
+	 *  multiple errors are suppressed).  This will be useful
+	 *  in a gui where you want to probably grey out tokens that are consumed
+	 *  but not matched to anything in grammar.  Anything between
+	 *  a beginResync/endResync pair was tossed out by the parser.
+	 */
+	public void endResync();
+
+	/** A semantic predicate was evaluate with this result and action text */
+	public void semanticPredicate(boolean result, String predicate);
+
+	/** Announce that parsing has begun.  Not technically useful except for
+	 *  sending events over a socket.  A GUI for example will launch a thread
+	 *  to connect and communicate with a remote parser.  The thread will want
+	 *  to notify the GUI when a connection is made.  ANTLR parsers
+	 *  trigger this upon entry to the first rule (the ruleLevel is used to
+	 *  figure this out).
+	 */
+	public void commence();
+
+	/** Parsing is over; successfully or not.  Mostly useful for telling
+	 *  remote debugging listeners that it's time to quit.  When the rule
+	 *  invocation level goes to zero at the end of a rule, we are done
+	 *  parsing.
+	 */
+	public void terminate();
+
+
+	// T r e e  P a r s i n g
+
+	/** Input for a tree parser is an AST, but we know nothing for sure
+	 *  about a node except its type and text (obtained from the adaptor).
+	 *  This is the analog of the consumeToken method.  Again, the ID is
+	 *  the hashCode usually of the node so it only works if hashCode is
+	 *  not implemented.  If the type is UP or DOWN, then
+	 *  the ID is not really meaningful as it's fixed--there is
+	 *  just one UP node and one DOWN navigation node.
+	 * @param t
+	 */
+	public void consumeNode(Object t);
+
+	/** The tree parser lookedahead.  If the type is UP or DOWN,
+	 *  then the ID is not really meaningful as it's fixed--there is
+	 *  just one UP node and one DOWN navigation node.
+	 */
+	public void LT(int i, Object t);
+
+
+	// A S T  E v e n t s
+
+	/** A nil was created (even nil nodes have a unique ID...
+	 *  they are not "null" per se).  As of 4/28/2006, this
+	 *  seems to be uniquely triggered when starting a new subtree
+	 *  such as when entering a subrule in automatic mode and when
+	 *  building a tree in rewrite mode.
+     *
+ 	 *  If you are receiving this event over a socket via
+	 *  RemoteDebugEventSocketListener then only t.ID is set.
+	 */
+	public void nilNode(Object t);
+
+	/** Upon syntax error, recognizers bracket the error with an error node
+	 *  if they are building ASTs.
+	 * @param t
+	 */
+	public void errorNode(Object t);
+
+	/** Announce a new node built from token elements such as type etc...
+	 * 
+	 *  If you are receiving this event over a socket via
+	 *  RemoteDebugEventSocketListener then only t.ID, type, text are
+	 *  set.
+	 */
+	public void createNode(Object t);
+
+	/** Announce a new node built from an existing token.
+	 *
+	 *  If you are receiving this event over a socket via
+	 *  RemoteDebugEventSocketListener then only node.ID and token.tokenIndex
+	 *  are set.
+	 */
+	public void createNode(Object node, Token token);
+
+	/** Make a node the new root of an existing root.  See
+	 *
+	 *  Note: the newRootID parameter is possibly different
+	 *  than the TreeAdaptor.becomeRoot() newRoot parameter.
+	 *  In our case, it will always be the result of calling
+	 *  TreeAdaptor.becomeRoot() and not root_n or whatever.
+	 *
+	 *  The listener should assume that this event occurs
+	 *  only when the current subrule (or rule) subtree is
+	 *  being reset to newRootID.
+	 * 
+	 *  If you are receiving this event over a socket via
+	 *  RemoteDebugEventSocketListener then only IDs are set.
+	 *
+	 *  @see org.antlr.runtime.tree.TreeAdaptor.becomeRoot()
+	 */
+	public void becomeRoot(Object newRoot, Object oldRoot);
+
+	/** Make childID a child of rootID.
+	 *
+	 *  If you are receiving this event over a socket via
+	 *  RemoteDebugEventSocketListener then only IDs are set.
+	 * 
+	 *  @see org.antlr.runtime.tree.TreeAdaptor.addChild()
+	 */
+	public void addChild(Object root, Object child);
+
+	/** Set the token start/stop token index for a subtree root or node.
+	 *
+	 *  If you are receiving this event over a socket via
+	 *  RemoteDebugEventSocketListener then only t.ID is set.
+	 */
+	public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex);
+}
diff --git a/src/org/antlr/runtime/debug/DebugEventRepeater.java b/src/org/antlr/runtime/debug/DebugEventRepeater.java
new file mode 100755
index 0000000..8fb6b66
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugEventRepeater.java
@@ -0,0 +1,88 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.RecognitionException;
+
+/** A simple event repeater (proxy) that delegates all functionality to the
+ *  listener sent into the ctor.  Useful if you want to listen in on a few
+ *  debug events w/o interrupting the debugger.  Just subclass the repeater
+ *  and override the methods you want to listen in on.  Remember to call
+ *  the method in this class so the event will continue on to the original
+ *  recipient.
+ *
+ *  @see DebugEventHub
+ */
+public class DebugEventRepeater implements DebugEventListener {
+	protected DebugEventListener listener;
+
+	public DebugEventRepeater(DebugEventListener listener) {
+		this.listener = listener;
+	}
+	
+	public void enterRule(String grammarFileName, String ruleName) { listener.enterRule(grammarFileName, ruleName); }
+	public void exitRule(String grammarFileName, String ruleName) { listener.exitRule(grammarFileName, ruleName); }
+	public void enterAlt(int alt) { listener.enterAlt(alt); }
+	public void enterSubRule(int decisionNumber) { listener.enterSubRule(decisionNumber); }
+	public void exitSubRule(int decisionNumber) { listener.exitSubRule(decisionNumber); }
+	public void enterDecision(int decisionNumber, boolean couldBacktrack) { listener.enterDecision(decisionNumber, couldBacktrack); }
+	public void exitDecision(int decisionNumber) { listener.exitDecision(decisionNumber); }
+	public void location(int line, int pos) { listener.location(line, pos); }
+	public void consumeToken(Token token) { listener.consumeToken(token); }
+	public void consumeHiddenToken(Token token) { listener.consumeHiddenToken(token); }
+	public void LT(int i, Token t) { listener.LT(i, t); }
+	public void mark(int i) { listener.mark(i); }
+	public void rewind(int i) { listener.rewind(i); }
+	public void rewind() { listener.rewind(); }
+	public void beginBacktrack(int level) { listener.beginBacktrack(level); }
+	public void endBacktrack(int level, boolean successful) { listener.endBacktrack(level, successful); }
+	public void recognitionException(RecognitionException e) { listener.recognitionException(e); }
+	public void beginResync() { listener.beginResync(); }
+	public void endResync() { listener.endResync(); }
+	public void semanticPredicate(boolean result, String predicate) { listener.semanticPredicate(result, predicate); }
+	public void commence() { listener.commence(); }
+	public void terminate() { listener.terminate(); }
+
+	// Tree parsing stuff
+
+	public void consumeNode(Object t) { listener.consumeNode(t); }
+	public void LT(int i, Object t) { listener.LT(i, t); }
+
+	// AST Stuff
+
+	public void nilNode(Object t) { listener.nilNode(t); }
+	public void errorNode(Object t) { listener.errorNode(t); }
+	public void createNode(Object t) { listener.createNode(t); }
+	public void createNode(Object node, Token token) { listener.createNode(node, token); }
+	public void becomeRoot(Object newRoot, Object oldRoot) { listener.becomeRoot(newRoot, oldRoot); }
+	public void addChild(Object root, Object child) { listener.addChild(root, child); }
+	public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex) {
+		listener.setTokenBoundaries(t, tokenStartIndex, tokenStopIndex);
+	}
+}
diff --git a/src/org/antlr/runtime/debug/DebugEventSocketProxy.java b/src/org/antlr/runtime/debug/DebugEventSocketProxy.java
new file mode 100755
index 0000000..3b480ad
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugEventSocketProxy.java
@@ -0,0 +1,354 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.BaseRecognizer;
+import org.antlr.runtime.tree.TreeAdaptor;
+
+import java.io.*;
+import java.net.ServerSocket;
+import java.net.Socket;
+
+/** A proxy debug event listener that forwards events over a socket to
+ *  a debugger (or any other listener) using a simple text-based protocol;
+ *  one event per line.  ANTLRWorks listens on server socket with a
+ *  RemoteDebugEventSocketListener instance.  These two objects must therefore
+ *  be kept in sync.  New events must be handled on both sides of socket.
+ */
+public class DebugEventSocketProxy extends BlankDebugEventListener {
+	public static final int DEFAULT_DEBUGGER_PORT = 49100; // was 49153
+	protected int port = DEFAULT_DEBUGGER_PORT;
+	protected ServerSocket serverSocket;
+	protected Socket socket;
+	protected String grammarFileName;
+	protected PrintWriter out;
+	protected BufferedReader in;
+
+	/** Who am i debugging? */
+	protected BaseRecognizer recognizer;
+
+	/** Almost certainly the recognizer will have adaptor set, but
+	 *  we don't know how to cast it (Parser or TreeParser) to get
+	 *  the adaptor field.  Must be set with a constructor. :(
+	 */
+	protected TreeAdaptor adaptor;
+
+	public DebugEventSocketProxy(BaseRecognizer recognizer, TreeAdaptor adaptor) {
+		this(recognizer, DEFAULT_DEBUGGER_PORT, adaptor);
+	}
+
+	public DebugEventSocketProxy(BaseRecognizer recognizer, int port, TreeAdaptor adaptor) {
+		this.grammarFileName = recognizer.getGrammarFileName();
+		this.adaptor = adaptor;
+		this.port = port;
+	}
+
+	public void handshake() throws IOException {
+		if ( serverSocket==null ) {
+			serverSocket = new ServerSocket(port);
+			socket = serverSocket.accept();
+			socket.setTcpNoDelay(true);
+			OutputStream os = socket.getOutputStream();
+			OutputStreamWriter osw = new OutputStreamWriter(os, "UTF8");
+			out = new PrintWriter(new BufferedWriter(osw));
+			InputStream is = socket.getInputStream();
+			InputStreamReader isr = new InputStreamReader(is, "UTF8");
+			in = new BufferedReader(isr);
+			out.println("ANTLR "+ DebugEventListener.PROTOCOL_VERSION);
+			out.println("grammar \""+ grammarFileName);
+			out.flush();
+			ack();
+		}
+	}
+
+	public void commence() {
+		// don't bother sending event; listener will trigger upon connection
+	}
+
+	public void terminate() {
+		transmit("terminate");
+		out.close();
+		try {
+			socket.close();
+		}
+		catch (IOException ioe) {
+			ioe.printStackTrace(System.err);
+		}
+	}
+
+	protected void ack() {
+		try {
+			in.readLine();
+		}
+		catch (IOException ioe) {
+			ioe.printStackTrace(System.err);
+		}
+	}
+
+	protected void transmit(String event) {
+		out.println(event);
+		out.flush();
+		ack();
+	}
+
+	public void enterRule(String grammarFileName, String ruleName) {
+		transmit("enterRule\t"+grammarFileName+"\t"+ruleName);
+	}
+
+	public void enterAlt(int alt) {
+		transmit("enterAlt\t"+alt);
+	}
+
+	public void exitRule(String grammarFileName, String ruleName) {
+		transmit("exitRule\t"+grammarFileName+"\t"+ruleName);
+	}
+
+	public void enterSubRule(int decisionNumber) {
+		transmit("enterSubRule\t"+decisionNumber);
+	}
+
+	public void exitSubRule(int decisionNumber) {
+		transmit("exitSubRule\t"+decisionNumber);
+	}
+
+	public void enterDecision(int decisionNumber, boolean couldBacktrack) {
+		transmit("enterDecision\t"+decisionNumber+"\t"+couldBacktrack);
+	}
+
+	public void exitDecision(int decisionNumber) {
+		transmit("exitDecision\t"+decisionNumber);
+	}
+
+	public void consumeToken(Token t) {
+		String buf = serializeToken(t);
+		transmit("consumeToken\t"+buf);
+	}
+
+	public void consumeHiddenToken(Token t) {
+		String buf = serializeToken(t);
+		transmit("consumeHiddenToken\t"+buf);
+	}
+
+	public void LT(int i, Token t) {
+        if(t != null)
+            transmit("LT\t"+i+"\t"+serializeToken(t));
+	}
+
+	public void mark(int i) {
+		transmit("mark\t"+i);
+	}
+
+	public void rewind(int i) {
+		transmit("rewind\t"+i);
+	}
+
+	public void rewind() {
+		transmit("rewind");
+	}
+
+	public void beginBacktrack(int level) {
+		transmit("beginBacktrack\t"+level);
+	}
+
+	public void endBacktrack(int level, boolean successful) {
+		transmit("endBacktrack\t"+level+"\t"+(successful?TRUE:FALSE));
+	}
+
+	public void location(int line, int pos) {
+		transmit("location\t"+line+"\t"+pos);
+	}
+
+	public void recognitionException(RecognitionException e) {
+		StringBuffer buf = new StringBuffer(50);
+		buf.append("exception\t");
+		buf.append(e.getClass().getName());
+		// dump only the data common to all exceptions for now
+		buf.append("\t");
+		buf.append(e.index);
+		buf.append("\t");
+		buf.append(e.line);
+		buf.append("\t");
+		buf.append(e.charPositionInLine);
+		transmit(buf.toString());
+	}
+
+	public void beginResync() {
+		transmit("beginResync");
+	}
+
+	public void endResync() {
+		transmit("endResync");
+	}
+
+	public void semanticPredicate(boolean result, String predicate) {
+		StringBuffer buf = new StringBuffer(50);
+		buf.append("semanticPredicate\t");
+		buf.append(result);
+		serializeText(buf, predicate);
+		transmit(buf.toString());
+	}
+
+	// A S T  P a r s i n g  E v e n t s
+
+	public void consumeNode(Object t) {
+		StringBuffer buf = new StringBuffer(50);
+		buf.append("consumeNode");
+		serializeNode(buf, t);
+		transmit(buf.toString());
+	}
+
+	public void LT(int i, Object t) {
+		int ID = adaptor.getUniqueID(t);
+		String text = adaptor.getText(t);
+		int type = adaptor.getType(t);
+		StringBuffer buf = new StringBuffer(50);
+		buf.append("LN\t"); // lookahead node; distinguish from LT in protocol
+		buf.append(i);
+		serializeNode(buf, t);
+		transmit(buf.toString());
+	}
+
+	protected void serializeNode(StringBuffer buf, Object t) {
+		int ID = adaptor.getUniqueID(t);
+		String text = adaptor.getText(t);
+		int type = adaptor.getType(t);
+		buf.append("\t");
+		buf.append(ID);
+		buf.append("\t");
+		buf.append(type);
+		Token token = adaptor.getToken(t);
+		int line = -1;
+		int pos = -1;
+		if ( token!=null ) {
+			line = token.getLine();
+			pos = token.getCharPositionInLine();
+		}
+		buf.append("\t");
+		buf.append(line);
+		buf.append("\t");
+		buf.append(pos);
+		int tokenIndex = adaptor.getTokenStartIndex(t);
+		buf.append("\t");
+		buf.append(tokenIndex);
+		serializeText(buf, text);
+	}
+
+
+	// A S T  E v e n t s
+
+	public void nilNode(Object t) {
+		int ID = adaptor.getUniqueID(t);
+		transmit("nilNode\t"+ID);
+	}
+
+	public void errorNode(Object t) {
+		int ID = adaptor.getUniqueID(t);
+		String text = t.toString();
+		StringBuffer buf = new StringBuffer(50);
+		buf.append("errorNode\t");
+		buf.append(ID);
+		buf.append("\t");
+		buf.append(Token.INVALID_TOKEN_TYPE);
+		serializeText(buf, text);
+		transmit(buf.toString());
+	}
+
+	public void createNode(Object t) {
+		int ID = adaptor.getUniqueID(t);
+		String text = adaptor.getText(t);
+		int type = adaptor.getType(t);
+		StringBuffer buf = new StringBuffer(50);
+		buf.append("createNodeFromTokenElements\t");
+		buf.append(ID);
+		buf.append("\t");
+		buf.append(type);
+		serializeText(buf, text);
+		transmit(buf.toString());
+	}
+
+	public void createNode(Object node, Token token) {
+		int ID = adaptor.getUniqueID(node);
+		int tokenIndex = token.getTokenIndex();
+		transmit("createNode\t"+ID+"\t"+tokenIndex);
+	}
+
+	public void becomeRoot(Object newRoot, Object oldRoot) {
+		int newRootID = adaptor.getUniqueID(newRoot);
+		int oldRootID = adaptor.getUniqueID(oldRoot);
+		transmit("becomeRoot\t"+newRootID+"\t"+oldRootID);
+	}
+
+	public void addChild(Object root, Object child) {
+		int rootID = adaptor.getUniqueID(root);
+		int childID = adaptor.getUniqueID(child);
+		transmit("addChild\t"+rootID+"\t"+childID);
+	}
+
+	public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex) {
+		int ID = adaptor.getUniqueID(t);
+		transmit("setTokenBoundaries\t"+ID+"\t"+tokenStartIndex+"\t"+tokenStopIndex);
+	}
+
+
+    // support
+
+    public void setTreeAdaptor(TreeAdaptor adaptor) { this.adaptor = adaptor; }
+    public TreeAdaptor getTreeAdaptor() { return adaptor; }
+
+    protected String serializeToken(Token t) {
+        StringBuffer buf = new StringBuffer(50);
+        buf.append(t.getTokenIndex()); buf.append('\t');
+        buf.append(t.getType()); buf.append('\t');
+        buf.append(t.getChannel()); buf.append('\t');
+        buf.append(t.getLine()); buf.append('\t');
+		buf.append(t.getCharPositionInLine());
+		serializeText(buf, t.getText());
+		return buf.toString();
+	}
+
+	protected void serializeText(StringBuffer buf, String text) {
+		buf.append("\t\"");
+		if ( text==null ) {
+			text = "";
+		}
+		// escape \n and \r all text for token appears to exist on one line
+		// this escape is slow but easy to understand
+		text = escapeNewlines(text);
+		buf.append(text);
+	}
+
+	protected String escapeNewlines(String txt) {
+		txt = txt.replaceAll("%","%25");   // escape all escape char ;)
+		txt = txt.replaceAll("\n","%0A");  // escape \n
+		txt = txt.replaceAll("\r","%0D");  // escape \r
+		return txt;
+	}
+}
+
diff --git a/src/org/antlr/runtime/debug/DebugParser.java b/src/org/antlr/runtime/debug/DebugParser.java
new file mode 100755
index 0000000..49d78e7
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugParser.java
@@ -0,0 +1,98 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.*;
+
+import java.io.IOException;
+
+public class DebugParser extends Parser {
+	/** Who to notify when events in the parser occur. */
+	protected DebugEventListener dbg = null;
+
+	/** Used to differentiate between fixed lookahead and cyclic DFA decisions
+	 *  while profiling.
+ 	 */
+	public boolean isCyclicDecision = false;
+
+	/** Create a normal parser except wrap the token stream in a debug
+	 *  proxy that fires consume events.
+	 */
+	public DebugParser(TokenStream input, DebugEventListener dbg, RecognizerSharedState state) {
+		super(input instanceof DebugTokenStream?input:new DebugTokenStream(input,dbg), state);
+		setDebugListener(dbg);
+	}
+
+	public DebugParser(TokenStream input, RecognizerSharedState state) {
+		super(input instanceof DebugTokenStream?input:new DebugTokenStream(input,null), state);
+	}
+
+	public DebugParser(TokenStream input, DebugEventListener dbg) {
+		this(input instanceof DebugTokenStream?input:new DebugTokenStream(input,dbg), dbg, null);
+	}
+
+	/** Provide a new debug event listener for this parser.  Notify the
+	 *  input stream too that it should send events to this listener.
+	 */
+	public void setDebugListener(DebugEventListener dbg) {
+		if ( input instanceof DebugTokenStream ) {
+			((DebugTokenStream)input).setDebugListener(dbg);
+		}
+		this.dbg = dbg;
+	}
+
+	public DebugEventListener getDebugListener() {
+		return dbg;
+	}
+
+	public void reportError(IOException e) {
+		System.err.println(e);
+		e.printStackTrace(System.err);
+	}
+
+	public void beginResync() {
+		dbg.beginResync();
+	}
+
+	public void endResync() {
+		dbg.endResync();
+	}
+
+	public void beginBacktrack(int level) {
+		dbg.beginBacktrack(level);
+	}
+
+	public void endBacktrack(int level, boolean successful) {
+		dbg.endBacktrack(level,successful);		
+	}
+
+	public void reportError(RecognitionException e) {
+		super.reportError(e);
+		dbg.recognitionException(e);
+	}
+}
diff --git a/src/org/antlr/runtime/debug/DebugTokenStream.java b/src/org/antlr/runtime/debug/DebugTokenStream.java
new file mode 100755
index 0000000..9a7a75f
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugTokenStream.java
@@ -0,0 +1,156 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.*;
+
+import java.util.List;
+
+public class DebugTokenStream implements TokenStream {
+	protected DebugEventListener dbg;
+	public TokenStream input;
+	protected boolean initialStreamState = true;
+
+	/** Track the last mark() call result value for use in rewind(). */
+	protected int lastMarker;
+
+	public DebugTokenStream(TokenStream input, DebugEventListener dbg) {
+		this.input = input;
+		setDebugListener(dbg);
+		// force TokenStream to get at least first valid token
+		// so we know if there are any hidden tokens first in the stream
+		input.LT(1);
+	}
+
+	public void setDebugListener(DebugEventListener dbg) {
+		this.dbg = dbg;
+	}
+
+	public void consume() {
+		if ( initialStreamState ) {
+			consumeInitialHiddenTokens();
+		}
+		int a = input.index();
+		Token t = input.LT(1);
+		input.consume();
+		int b = input.index();
+		dbg.consumeToken(t);
+		if ( b>a+1 ) {
+			// then we consumed more than one token; must be off channel tokens
+			for (int i=a+1; i<b; i++) {
+				dbg.consumeHiddenToken(input.get(i));
+			}
+		}
+	}
+
+	/* consume all initial off-channel tokens */
+	protected void consumeInitialHiddenTokens() {
+		int firstOnChannelTokenIndex = input.index();
+		for (int i=0; i<firstOnChannelTokenIndex; i++) {
+			dbg.consumeHiddenToken(input.get(i));
+		}
+		initialStreamState = false;
+	}
+
+	public Token LT(int i) {
+		if ( initialStreamState ) {
+			consumeInitialHiddenTokens();
+		}
+		dbg.LT(i, input.LT(i));
+		return input.LT(i);
+	}
+
+	public int LA(int i) {
+		if ( initialStreamState ) {
+			consumeInitialHiddenTokens();
+		}
+		dbg.LT(i, input.LT(i));
+		return input.LA(i);
+	}
+
+	public Token get(int i) {
+		return input.get(i);
+	}
+
+	public int mark() {
+		lastMarker = input.mark();
+		dbg.mark(lastMarker);
+		return lastMarker;
+	}
+
+	public int index() {
+		return input.index();
+	}
+
+	public int range() {
+		return input.range();
+	}
+
+	public void rewind(int marker) {
+		dbg.rewind(marker);
+		input.rewind(marker);
+	}
+
+	public void rewind() {
+		dbg.rewind();
+		input.rewind(lastMarker);
+	}
+
+	public void release(int marker) {
+	}
+
+	public void seek(int index) {
+		// TODO: implement seek in dbg interface
+		// db.seek(index);
+		input.seek(index);
+	}
+
+	public int size() {
+		return input.size();
+	}
+
+	public TokenSource getTokenSource() {
+		return input.getTokenSource();
+	}
+
+	public String getSourceName() {
+		return getTokenSource().getSourceName();
+	}
+
+	public String toString() {
+		return input.toString();
+	}
+
+	public String toString(int start, int stop) {
+		return input.toString(start,stop);
+	}
+
+	public String toString(Token start, Token stop) {
+		return input.toString(start,stop);
+	}
+}
diff --git a/src/org/antlr/runtime/debug/DebugTreeAdaptor.java b/src/org/antlr/runtime/debug/DebugTreeAdaptor.java
new file mode 100755
index 0000000..c72a2b6
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugTreeAdaptor.java
@@ -0,0 +1,250 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.tree.TreeAdaptor;
+
+/** A TreeAdaptor proxy that fires debugging events to a DebugEventListener
+ *  delegate and uses the TreeAdaptor delegate to do the actual work.  All
+ *  AST events are triggered by this adaptor; no code gen changes are needed
+ *  in generated rules.  Debugging events are triggered *after* invoking
+ *  tree adaptor routines.
+ *
+ *  Trees created with actions in rewrite actions like "-> ^(ADD {foo} {bar})"
+ *  cannot be tracked as they might not use the adaptor to create foo, bar.
+ *  The debug listener has to deal with tree node IDs for which it did
+ *  not see a createNode event.  A single <unknown> node is sufficient even
+ *  if it represents a whole tree.
+ */
+public class DebugTreeAdaptor implements TreeAdaptor {
+	protected DebugEventListener dbg;
+	protected TreeAdaptor adaptor;
+
+	public DebugTreeAdaptor(DebugEventListener dbg, TreeAdaptor adaptor) {
+		this.dbg = dbg;
+		this.adaptor = adaptor;
+	}
+
+	public Object create(Token payload) {
+		if ( payload.getTokenIndex() < 0 ) {
+			// could be token conjured up during error recovery
+			return create(payload.getType(), payload.getText());
+		}
+		Object node = adaptor.create(payload);
+		dbg.createNode(node, payload);
+		return node;
+	}
+
+	public Object errorNode(TokenStream input, Token start, Token stop,
+							RecognitionException e)
+	{
+		Object node = adaptor.errorNode(input, start, stop, e);
+		if ( node!=null ) {
+			dbg.errorNode(node);
+		}
+		return node;
+	}
+
+	public Object dupTree(Object tree) {
+		Object t = adaptor.dupTree(tree);
+		// walk the tree and emit create and add child events
+		// to simulate what dupTree has done. dupTree does not call this debug
+		// adapter so I must simulate.
+		simulateTreeConstruction(t);
+		return t;
+	}
+
+	/** ^(A B C): emit create A, create B, add child, ...*/
+	protected void simulateTreeConstruction(Object t) {
+		dbg.createNode(t);
+		int n = adaptor.getChildCount(t);
+		for (int i=0; i<n; i++) {
+			Object child = adaptor.getChild(t, i);
+			simulateTreeConstruction(child);
+			dbg.addChild(t, child);
+		}
+	}
+
+	public Object dupNode(Object treeNode) {
+		Object d = adaptor.dupNode(treeNode);
+		dbg.createNode(d);
+		return d;
+	}
+
+	public Object nil() {
+		Object node = adaptor.nil();
+		dbg.nilNode(node);
+		return node;
+	}
+
+	public boolean isNil(Object tree) {
+		return adaptor.isNil(tree);
+	}
+
+	public void addChild(Object t, Object child) {
+		if ( t==null || child==null ) {
+			return;
+		}
+		adaptor.addChild(t,child);
+		dbg.addChild(t, child);
+	}
+
+	public Object becomeRoot(Object newRoot, Object oldRoot) {
+		Object n = adaptor.becomeRoot(newRoot, oldRoot);
+		dbg.becomeRoot(newRoot, oldRoot);
+		return n;
+	}
+
+	public Object rulePostProcessing(Object root) {
+		return adaptor.rulePostProcessing(root);
+	}
+
+	public void addChild(Object t, Token child) {
+		Object n = this.create(child);
+		this.addChild(t, n);
+	}
+
+	public Object becomeRoot(Token newRoot, Object oldRoot) {
+		Object n = this.create(newRoot);
+		adaptor.becomeRoot(n, oldRoot);
+		dbg.becomeRoot(newRoot, oldRoot);
+		return n;
+	}
+
+	public Object create(int tokenType, Token fromToken) {
+		Object node = adaptor.create(tokenType, fromToken);
+		dbg.createNode(node);
+		return node;
+	}
+
+	public Object create(int tokenType, Token fromToken, String text) {
+		Object node = adaptor.create(tokenType, fromToken, text);
+		dbg.createNode(node);
+		return node;
+	}
+
+	public Object create(int tokenType, String text) {
+		Object node = adaptor.create(tokenType, text);
+		dbg.createNode(node);
+		return node;
+	}
+
+	public int getType(Object t) {
+		return adaptor.getType(t);
+	}
+
+	public void setType(Object t, int type) {
+		adaptor.setType(t, type);
+	}
+
+	public String getText(Object t) {
+		return adaptor.getText(t);
+	}
+
+	public void setText(Object t, String text) {
+		adaptor.setText(t, text);
+	}
+
+	public Token getToken(Object t) {
+		return adaptor.getToken(t);
+	}
+
+	public void setTokenBoundaries(Object t, Token startToken, Token stopToken) {
+		adaptor.setTokenBoundaries(t, startToken, stopToken);
+		if ( t!=null && startToken!=null && stopToken!=null ) {
+			dbg.setTokenBoundaries(
+				t, startToken.getTokenIndex(),
+				stopToken.getTokenIndex());
+		}
+	}
+
+	public int getTokenStartIndex(Object t) {
+		return adaptor.getTokenStartIndex(t);
+	}
+
+	public int getTokenStopIndex(Object t) {
+		return adaptor.getTokenStopIndex(t);
+	}
+
+	public Object getChild(Object t, int i) {
+		return adaptor.getChild(t, i);
+	}
+
+	public void setChild(Object t, int i, Object child) {
+		adaptor.setChild(t, i, child);
+	}
+
+	public Object deleteChild(Object t, int i) {
+		return deleteChild(t, i);
+	}
+
+	public int getChildCount(Object t) {
+		return adaptor.getChildCount(t);
+	}
+
+	public int getUniqueID(Object node) {
+		return adaptor.getUniqueID(node);
+	}
+
+	public Object getParent(Object t) {
+		return adaptor.getParent(t);
+	}
+
+	public int getChildIndex(Object t) {
+		return adaptor.getChildIndex(t);
+	}
+
+	public void setParent(Object t, Object parent) {
+		adaptor.setParent(t, parent);
+	}
+
+	public void setChildIndex(Object t, int index) {
+		adaptor.setChildIndex(t, index);
+	}
+
+	public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t) {
+		adaptor.replaceChildren(parent, startChildIndex, stopChildIndex, t);
+	}
+
+	// support
+
+	public DebugEventListener getDebugListener() {
+		return dbg;
+	}
+
+	public void setDebugListener(DebugEventListener dbg) {
+		this.dbg = dbg;
+	}
+
+	public TreeAdaptor getTreeAdaptor() {
+		return adaptor;
+	}
+}
diff --git a/src/org/antlr/runtime/debug/DebugTreeNodeStream.java b/src/org/antlr/runtime/debug/DebugTreeNodeStream.java
new file mode 100755
index 0000000..92ff009
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugTreeNodeStream.java
@@ -0,0 +1,155 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.tree.TreeAdaptor;
+import org.antlr.runtime.tree.TreeNodeStream;
+import org.antlr.runtime.TokenStream;
+
+/** Debug any tree node stream.  The constructor accepts the stream
+ *  and a debug listener.  As node stream calls come in, debug events
+ *  are triggered.
+ */
+public class DebugTreeNodeStream implements TreeNodeStream {
+	protected DebugEventListener dbg;
+	protected TreeAdaptor adaptor;
+	protected TreeNodeStream input;
+	protected boolean initialStreamState = true;
+
+	/** Track the last mark() call result value for use in rewind(). */
+	protected int lastMarker;
+
+	public DebugTreeNodeStream(TreeNodeStream input,
+							   DebugEventListener dbg)
+	{
+		this.input = input;
+		this.adaptor = input.getTreeAdaptor();
+		this.input.setUniqueNavigationNodes(true);
+		setDebugListener(dbg);
+	}
+
+	public void setDebugListener(DebugEventListener dbg) {
+		this.dbg = dbg;
+	}
+
+	public TreeAdaptor getTreeAdaptor() {
+		return adaptor;
+	}
+
+	public void consume() {
+		Object node = input.LT(1);
+		input.consume();
+		dbg.consumeNode(node);
+	}
+
+	public Object get(int i) {
+		return input.get(i);
+	}
+
+	public Object LT(int i) {
+		Object node = input.LT(i);
+		int ID = adaptor.getUniqueID(node);
+		String text = adaptor.getText(node);
+		int type = adaptor.getType(node);
+		dbg.LT(i, node);
+		return node;
+	}
+
+	public int LA(int i) {
+		Object node = input.LT(i);
+		int ID = adaptor.getUniqueID(node);
+		String text = adaptor.getText(node);
+		int type = adaptor.getType(node);
+		dbg.LT(i, node);
+		return type;
+	}
+
+	public int mark() {
+		lastMarker = input.mark();
+		dbg.mark(lastMarker);
+		return lastMarker;
+	}
+
+	public int index() {
+		return input.index();
+	}
+
+	public void rewind(int marker) {
+		dbg.rewind(marker);
+		input.rewind(marker);
+	}
+
+	public void rewind() {
+		dbg.rewind();
+		input.rewind(lastMarker);
+	}
+
+	public void release(int marker) {
+	}
+
+	public void seek(int index) {
+		// TODO: implement seek in dbg interface
+		// db.seek(index);
+		input.seek(index);
+	}
+
+	public int size() {
+		return input.size();
+	}
+
+    public void reset() { ; }
+
+    public Object getTreeSource() {
+		return input;
+	}
+
+	public String getSourceName() {
+		return getTokenStream().getSourceName();
+	}
+
+	public TokenStream getTokenStream() {
+		return input.getTokenStream();
+	}
+
+	/** It is normally this object that instructs the node stream to
+	 *  create unique nav nodes, but to satisfy interface, we have to
+	 *  define it.  It might be better to ignore the parameter but
+	 *  there might be a use for it later, so I'll leave.
+	 */
+	public void setUniqueNavigationNodes(boolean uniqueNavigationNodes) {
+		input.setUniqueNavigationNodes(uniqueNavigationNodes);
+	}
+
+	public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t) {
+		input.replaceChildren(parent, startChildIndex, stopChildIndex, t);
+	}
+
+	public String toString(Object start, Object stop) {
+		return input.toString(start,stop);
+	}
+}
diff --git a/src/org/antlr/runtime/debug/DebugTreeParser.java b/src/org/antlr/runtime/debug/DebugTreeParser.java
new file mode 100755
index 0000000..6e1ece8
--- /dev/null
+++ b/src/org/antlr/runtime/debug/DebugTreeParser.java
@@ -0,0 +1,109 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.*;
+import org.antlr.runtime.tree.TreeNodeStream;
+import org.antlr.runtime.tree.TreeParser;
+
+import java.io.IOException;
+
+public class DebugTreeParser extends TreeParser {
+	/** Who to notify when events in the parser occur. */
+	protected DebugEventListener dbg = null;
+
+	/** Used to differentiate between fixed lookahead and cyclic DFA decisions
+	 *  while profiling.
+ 	 */
+	public boolean isCyclicDecision = false;
+
+	/** Create a normal parser except wrap the token stream in a debug
+	 *  proxy that fires consume events.
+	 */
+	public DebugTreeParser(TreeNodeStream input, DebugEventListener dbg, RecognizerSharedState state) {
+		super(input instanceof DebugTreeNodeStream?input:new DebugTreeNodeStream(input,dbg), state);
+		setDebugListener(dbg);
+	}
+
+	public DebugTreeParser(TreeNodeStream input, RecognizerSharedState state) {
+		super(input instanceof DebugTreeNodeStream?input:new DebugTreeNodeStream(input,null), state);
+	}
+
+	public DebugTreeParser(TreeNodeStream input, DebugEventListener dbg) {
+		this(input instanceof DebugTreeNodeStream?input:new DebugTreeNodeStream(input,dbg), dbg, null);
+	}
+
+	/** Provide a new debug event listener for this parser.  Notify the
+	 *  input stream too that it should send events to this listener.
+	 */
+	public void setDebugListener(DebugEventListener dbg) {
+		if ( input instanceof DebugTreeNodeStream ) {
+			((DebugTreeNodeStream)input).setDebugListener(dbg);
+		}
+		this.dbg = dbg;
+	}
+
+	public DebugEventListener getDebugListener() {
+		return dbg;
+	}
+
+	public void reportError(IOException e) {
+		System.err.println(e);
+		e.printStackTrace(System.err);
+	}
+
+	public void reportError(RecognitionException e) {
+		dbg.recognitionException(e);
+	}
+
+	protected Object getMissingSymbol(IntStream input,
+									  RecognitionException e,
+									  int expectedTokenType,
+									  BitSet follow)
+	{
+		Object o = super.getMissingSymbol(input, e, expectedTokenType, follow);
+		dbg.consumeNode(o);
+		return o;
+	}
+
+	public void beginResync() {
+		dbg.beginResync();
+	}
+
+	public void endResync() {
+		dbg.endResync();
+	}
+
+	public void beginBacktrack(int level) {
+		dbg.beginBacktrack(level);
+	}
+
+	public void endBacktrack(int level, boolean successful) {
+		dbg.endBacktrack(level,successful);		
+	}
+}
diff --git a/src/org/antlr/runtime/debug/ParseTreeBuilder.java b/src/org/antlr/runtime/debug/ParseTreeBuilder.java
new file mode 100755
index 0000000..13c6ed0
--- /dev/null
+++ b/src/org/antlr/runtime/debug/ParseTreeBuilder.java
@@ -0,0 +1,109 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.tree.ParseTree;
+
+import java.util.Stack;
+import java.util.ArrayList;
+import java.util.List;
+
+/** This parser listener tracks rule entry/exit and token matches
+ *  to build a simple parse tree using ParseTree nodes.
+ */
+public class ParseTreeBuilder extends BlankDebugEventListener {
+	public static final String EPSILON_PAYLOAD = "<epsilon>";
+	
+	Stack callStack = new Stack();
+	List hiddenTokens = new ArrayList();
+	int backtracking = 0;
+
+	public ParseTreeBuilder(String grammarName) {
+		ParseTree root = create("<grammar "+grammarName+">");
+		callStack.push(root);
+	}
+
+	public ParseTree getTree() {
+		return (ParseTree)callStack.elementAt(0);
+	}
+
+	/**  What kind of node to create.  You might want to override
+	 *   so I factored out creation here.
+	 */
+	public ParseTree create(Object payload) {
+		return new ParseTree(payload);
+	}
+
+	public ParseTree epsilonNode() {
+		return create(EPSILON_PAYLOAD);
+	}
+
+	/** Backtracking or cyclic DFA, don't want to add nodes to tree */
+	public void enterDecision(int d, boolean couldBacktrack) { backtracking++; }
+	public void exitDecision(int i) { backtracking--; }
+
+	public void enterRule(String filename, String ruleName) {
+		if ( backtracking>0 ) return;
+		ParseTree parentRuleNode = (ParseTree)callStack.peek();
+		ParseTree ruleNode = create(ruleName);
+		parentRuleNode.addChild(ruleNode);
+		callStack.push(ruleNode);
+	}
+
+	public void exitRule(String filename, String ruleName) {
+		if ( backtracking>0 ) return;
+		ParseTree ruleNode = (ParseTree)callStack.peek();
+		if ( ruleNode.getChildCount()==0 ) {
+			ruleNode.addChild(epsilonNode());
+		}
+		callStack.pop();		
+	}
+
+	public void consumeToken(Token token) {
+		if ( backtracking>0 ) return;
+		ParseTree ruleNode = (ParseTree)callStack.peek();
+		ParseTree elementNode = create(token);
+		elementNode.hiddenTokens = this.hiddenTokens;
+		this.hiddenTokens = new ArrayList();
+		ruleNode.addChild(elementNode);
+	}
+
+	public void consumeHiddenToken(Token token) {
+		if ( backtracking>0 ) return;
+		hiddenTokens.add(token);
+	}
+
+	public void recognitionException(RecognitionException e) {
+		if ( backtracking>0 ) return;
+		ParseTree ruleNode = (ParseTree)callStack.peek();
+		ParseTree errorNode = create(e);
+		ruleNode.addChild(errorNode);
+	}
+}
diff --git a/src/org/antlr/runtime/debug/Profiler.java b/src/org/antlr/runtime/debug/Profiler.java
new file mode 100755
index 0000000..aea9a17
--- /dev/null
+++ b/src/org/antlr/runtime/debug/Profiler.java
@@ -0,0 +1,734 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.*;
+import org.antlr.runtime.misc.DoubleKeyMap;
+
+import java.util.*;
+
+/** Using the debug event interface, track what is happening in the parser
+ *  and record statistics about the runtime.
+ */
+public class Profiler extends BlankDebugEventListener {
+	public static final String DATA_SEP = "\t";
+	public static final String newline = System.getProperty("line.separator");
+
+	static boolean dump = false;
+
+	public static class ProfileStats {
+		public String Version;
+		public String name;
+		public int numRuleInvocations;
+		public int numUniqueRulesInvoked;
+		public int numDecisionEvents;
+		public int numDecisionsCovered;
+		public int numDecisionsThatPotentiallyBacktrack;
+		public int numDecisionsThatDoBacktrack;
+		public int maxRuleInvocationDepth;
+		public float avgkPerDecisionEvent;
+		public float avgkPerBacktrackingDecisionEvent;
+		public float averageDecisionPercentBacktracks;
+		public int numBacktrackOccurrences; // doesn't count gated DFA edges
+
+		public int numFixedDecisions;
+		public int minDecisionMaxFixedLookaheads;
+		public int maxDecisionMaxFixedLookaheads;
+		public int avgDecisionMaxFixedLookaheads;
+		public int stddevDecisionMaxFixedLookaheads;
+		public int numCyclicDecisions;
+		public int minDecisionMaxCyclicLookaheads;
+		public int maxDecisionMaxCyclicLookaheads;
+		public int avgDecisionMaxCyclicLookaheads;
+		public int stddevDecisionMaxCyclicLookaheads;
+//		int Stats.min(toArray(decisionMaxSynPredLookaheads);
+//		int Stats.max(toArray(decisionMaxSynPredLookaheads);
+//		int Stats.avg(toArray(decisionMaxSynPredLookaheads);
+//		int Stats.stddev(toArray(decisionMaxSynPredLookaheads);
+		public int numSemanticPredicates;
+		public int numTokens;
+		public int numHiddenTokens;
+		public int numCharsMatched;
+		public int numHiddenCharsMatched;
+		public int numReportedErrors;
+		public int numMemoizationCacheHits;
+		public int numMemoizationCacheMisses;
+		public int numGuessingRuleInvocations;
+		public int numMemoizationCacheEntries;
+	}
+
+	public static class DecisionDescriptor {
+		public int decision;
+		public String fileName;
+		public String ruleName;
+		public int line;
+		public int pos;
+		public boolean couldBacktrack;
+
+		public int n;
+		public float avgk; // avg across all decision events
+		public int maxk;
+		public int numBacktrackOccurrences;
+		public int numSemPredEvals;
+	}
+
+	// all about a specific exec of a single decision
+	public static class DecisionEvent {
+		public DecisionDescriptor decision;
+		public int startIndex;
+		public int k;
+		public boolean backtracks; // doesn't count gated DFA edges
+		public boolean evalSemPred;
+		public long startTime;
+		public long stopTime;
+		public int numMemoizationCacheHits;
+		public int numMemoizationCacheMisses;
+	}
+
+	/** Because I may change the stats, I need to track that for later
+	 *  computations to be consistent.
+	 */
+	public static final String Version = "3";
+	public static final String RUNTIME_STATS_FILENAME = "runtime.stats";
+
+	/** Ack, should not store parser; can't do remote stuff.  Well, we pass
+	 *  input stream around too so I guess it's ok.
+	 */
+	public DebugParser parser = null;
+
+	// working variables
+
+	protected int ruleLevel = 0;
+	//protected int decisionLevel = 0;
+	protected Token lastRealTokenTouchedInDecision;
+	protected Set<String> uniqueRules = new HashSet<String>();
+	protected Stack<String> currentGrammarFileName = new Stack();
+	protected Stack<String> currentRuleName = new Stack();
+	protected Stack<Integer> currentLine = new Stack();
+	protected Stack<Integer> currentPos = new Stack();
+
+	// Vector<DecisionStats>
+	//protected Vector decisions = new Vector(200); // need setSize
+	protected DoubleKeyMap<String,Integer, DecisionDescriptor> decisions =
+		new DoubleKeyMap<String,Integer, DecisionDescriptor>();
+
+	// Record a DecisionData for each decision we hit while parsing
+	protected List<DecisionEvent> decisionEvents = new ArrayList<DecisionEvent>();
+	protected Stack<DecisionEvent> decisionStack = new Stack<DecisionEvent>();
+
+	protected int backtrackDepth;
+	
+	ProfileStats stats = new ProfileStats();
+
+	public Profiler() {
+	}
+
+	public Profiler(DebugParser parser) {
+		this.parser = parser;
+	}
+
+	public void enterRule(String grammarFileName, String ruleName) {
+//		System.out.println("enterRule "+grammarFileName+":"+ruleName);
+		ruleLevel++;
+		stats.numRuleInvocations++;
+		uniqueRules.add(grammarFileName+":"+ruleName);
+		stats.maxRuleInvocationDepth = Math.max(stats.maxRuleInvocationDepth, ruleLevel);
+		currentGrammarFileName.push( grammarFileName );
+		currentRuleName.push( ruleName );
+	}
+
+	public void exitRule(String grammarFileName, String ruleName) {
+		ruleLevel--;
+		currentGrammarFileName.pop();
+		currentRuleName.pop();
+	}
+
+	/** Track memoization; this is not part of standard debug interface
+	 *  but is triggered by profiling.  Code gen inserts an override
+	 *  for this method in the recognizer, which triggers this method.
+	 *  Called from alreadyParsedRule().
+	 */
+	public void examineRuleMemoization(IntStream input,
+									   int ruleIndex,
+									   int stopIndex, // index or MEMO_RULE_UNKNOWN...
+									   String ruleName)
+	{
+		if (dump) System.out.println("examine memo "+ruleName+" at "+input.index()+": "+stopIndex);
+		if ( stopIndex==BaseRecognizer.MEMO_RULE_UNKNOWN ) {
+			//System.out.println("rule "+ruleIndex+" missed @ "+input.index());
+			stats.numMemoizationCacheMisses++;
+			stats.numGuessingRuleInvocations++; // we'll have to enter
+			currentDecision().numMemoizationCacheMisses++;
+		}
+		else {
+			// regardless of rule success/failure, if in cache, we have a cache hit
+			//System.out.println("rule "+ruleIndex+" hit @ "+input.index());
+			stats.numMemoizationCacheHits++;
+			currentDecision().numMemoizationCacheHits++;
+		}
+	}
+
+	/** Warning: doesn't track success/failure, just unique recording event */
+	public void memoize(IntStream input,
+						int ruleIndex,
+						int ruleStartIndex,
+						String ruleName)
+	{
+		// count how many entries go into table
+		if (dump) System.out.println("memoize "+ruleName);
+		stats.numMemoizationCacheEntries++;
+	}
+
+	@Override
+	public void location(int line, int pos) {
+		currentLine.push(line);
+		currentPos.push(pos);
+	}
+
+	public void enterDecision(int decisionNumber, boolean couldBacktrack) {
+		lastRealTokenTouchedInDecision = null;
+		stats.numDecisionEvents++;
+		int startingLookaheadIndex = parser.getTokenStream().index();
+		TokenStream input = parser.getTokenStream();
+		if ( dump ) System.out.println("enterDecision canBacktrack="+couldBacktrack+" "+ decisionNumber +
+						   " backtrack depth " + backtrackDepth +
+						   " @ " + input.get(input.index()) +
+						   " rule " +locationDescription());
+		String g = (String) currentGrammarFileName.peek();
+		DecisionDescriptor descriptor = decisions.get(g, decisionNumber);
+		if ( descriptor == null ) {
+			descriptor = new DecisionDescriptor();
+			decisions.put(g, decisionNumber, descriptor);
+			descriptor.decision = decisionNumber;
+			descriptor.fileName = (String)currentGrammarFileName.peek();
+			descriptor.ruleName = (String)currentRuleName.peek();
+			descriptor.line = (Integer)currentLine.peek();
+			descriptor.pos = (Integer)currentPos.peek();
+			descriptor.couldBacktrack = couldBacktrack;
+		}
+		descriptor.n++;
+
+		DecisionEvent d = new DecisionEvent();
+		decisionStack.push(d);
+		d.decision = descriptor;
+		d.startTime = System.currentTimeMillis();
+		d.startIndex = startingLookaheadIndex;
+	}
+
+	public void exitDecision(int decisionNumber) {
+		DecisionEvent d = decisionStack.pop();
+		d.stopTime = System.currentTimeMillis();
+
+		int lastTokenIndex = lastRealTokenTouchedInDecision.getTokenIndex();
+		int numHidden = getNumberOfHiddenTokens(d.startIndex, lastTokenIndex);
+		int depth = lastTokenIndex - d.startIndex - numHidden + 1; // +1 counts consuming start token as 1
+		d.k = depth;
+		d.decision.maxk = Math.max(d.decision.maxk, depth);
+
+		if (dump) System.out.println("exitDecision "+decisionNumber+" in "+d.decision.ruleName+
+						   " lookahead "+d.k +" max token "+lastRealTokenTouchedInDecision);
+		decisionEvents.add(d); // done with decision; track all
+	}
+
+	public void consumeToken(Token token) {
+		if (dump) System.out.println("consume token "+token);
+		if ( !inDecision() ) {
+			stats.numTokens++;
+			return;
+		}
+		if ( lastRealTokenTouchedInDecision==null ||
+			 lastRealTokenTouchedInDecision.getTokenIndex() < token.getTokenIndex() )
+		{
+			lastRealTokenTouchedInDecision = token;
+		}
+		DecisionEvent d = currentDecision();
+		// compute lookahead depth
+		int thisRefIndex = token.getTokenIndex();
+		int numHidden = getNumberOfHiddenTokens(d.startIndex, thisRefIndex);
+		int depth = thisRefIndex - d.startIndex - numHidden + 1; // +1 counts consuming start token as 1
+		//d.maxk = Math.max(d.maxk, depth);
+		if (dump) System.out.println("consume "+thisRefIndex+" "+depth+" tokens ahead in "+
+						   d.decision.ruleName+"-"+d.decision.decision+" start index "+d.startIndex);		
+	}
+
+	/** The parser is in a decision if the decision depth > 0.  This
+	 *  works for backtracking also, which can have nested decisions.
+	 */
+	public boolean inDecision() {
+		return decisionStack.size()>0;
+	}
+
+	public void consumeHiddenToken(Token token) {
+		//System.out.println("consume hidden token "+token);
+		if ( !inDecision() ) stats.numHiddenTokens++;
+	}
+
+	/** Track refs to lookahead if in a fixed/nonfixed decision.
+	 */
+	public void LT(int i, Token t) {
+		if ( inDecision() && i>0 ) {
+			DecisionEvent d = currentDecision();
+			if (dump) System.out.println("LT("+i+")="+t+" index "+t.getTokenIndex()+" relative to "+d.decision.ruleName+"-"+
+							   d.decision.decision+" start index "+d.startIndex);
+			if ( lastRealTokenTouchedInDecision==null ||
+				 lastRealTokenTouchedInDecision.getTokenIndex() < t.getTokenIndex() )
+			{
+				lastRealTokenTouchedInDecision = t;
+				if (dump) System.out.println("set last token "+lastRealTokenTouchedInDecision);
+			}
+			// get starting index off stack
+//			int stackTop = lookaheadStack.size()-1;
+//			Integer startingIndex = (Integer)lookaheadStack.get(stackTop);
+//			// compute lookahead depth
+//			int thisRefIndex = parser.getTokenStream().index();
+//			int numHidden =
+//				getNumberOfHiddenTokens(startingIndex.intValue(), thisRefIndex);
+//			int depth = i + thisRefIndex - startingIndex.intValue() - numHidden;
+//			/*
+//			System.out.println("LT("+i+") @ index "+thisRefIndex+" is depth "+depth+
+//				" max is "+maxLookaheadInCurrentDecision);
+//			*/
+//			if ( depth>maxLookaheadInCurrentDecision ) {
+//				maxLookaheadInCurrentDecision = depth;
+//			}
+//			d.maxk = currentDecision()/
+		}
+	}
+
+	/** Track backtracking decisions.  You'll see a fixed or cyclic decision
+	 *  and then a backtrack.
+	 *
+	 * 		enter rule
+	 * 		...
+	 * 		enter decision
+	 * 		LA and possibly consumes (for cyclic DFAs)
+	 * 		begin backtrack level
+	 * 		mark m
+	 * 		rewind m
+	 * 		end backtrack level, success
+	 * 		exit decision
+	 * 		...
+	 * 		exit rule
+	 */
+	public void beginBacktrack(int level) {
+		if (dump) System.out.println("enter backtrack "+level);
+		backtrackDepth++;
+		DecisionEvent e = currentDecision();
+		if ( e.decision.couldBacktrack ) {
+			stats.numBacktrackOccurrences++;
+			e.decision.numBacktrackOccurrences++;
+			e.backtracks = true;
+		}
+	}
+
+	/** Successful or not, track how much lookahead synpreds use */
+	public void endBacktrack(int level, boolean successful) {
+		if (dump) System.out.println("exit backtrack "+level+": "+successful);
+		backtrackDepth--;		
+	}
+
+	@Override
+	public void mark(int i) {
+		if (dump) System.out.println("mark "+i);
+	}
+
+	@Override
+	public void rewind(int i) {
+		if (dump) System.out.println("rewind "+i);
+	}
+
+	@Override
+	public void rewind() {
+		if (dump) System.out.println("rewind");
+	}
+
+
+
+	protected DecisionEvent currentDecision() {
+		return decisionStack.peek();
+	}
+
+	public void recognitionException(RecognitionException e) {
+		stats.numReportedErrors++;
+	}
+
+	public void semanticPredicate(boolean result, String predicate) {
+		stats.numSemanticPredicates++;
+		if ( inDecision() ) {
+			DecisionEvent d = currentDecision();
+			d.evalSemPred = true;
+			d.decision.numSemPredEvals++;
+			if (dump) System.out.println("eval "+predicate+" in "+d.decision.ruleName+"-"+
+							   d.decision.decision);
+		}
+	}
+
+	public void terminate() {
+		for (DecisionEvent e : decisionEvents) {
+			//System.out.println("decision "+e.decision.decision+": k="+e.k);
+			e.decision.avgk += e.k;
+			stats.avgkPerDecisionEvent += e.k;
+			if ( e.backtracks ) { // doesn't count gated syn preds on DFA edges
+				stats.avgkPerBacktrackingDecisionEvent += e.k;
+			}
+		}
+		stats.averageDecisionPercentBacktracks = 0.0f;
+		for (DecisionDescriptor d : decisions.values()) {
+			stats.numDecisionsCovered++;
+			d.avgk /= (double)d.n;
+			if ( d.couldBacktrack ) {
+				stats.numDecisionsThatPotentiallyBacktrack++;
+				float percentBacktracks = d.numBacktrackOccurrences / (float)d.n;
+				//System.out.println("dec "+d.decision+" backtracks "+percentBacktracks*100+"%");
+				stats.averageDecisionPercentBacktracks += percentBacktracks;
+			}
+			// ignore rules that backtrack along gated DFA edges
+			if ( d.numBacktrackOccurrences > 0 ) {
+				stats.numDecisionsThatDoBacktrack++;
+			}
+		}
+		stats.averageDecisionPercentBacktracks /= stats.numDecisionsThatPotentiallyBacktrack;
+		stats.averageDecisionPercentBacktracks *= 100; // it's a percentage
+		stats.avgkPerDecisionEvent /= stats.numDecisionEvents;
+		stats.avgkPerBacktrackingDecisionEvent /= (double)stats.numBacktrackOccurrences;
+
+		System.err.println(toString());
+		System.err.println(getDecisionStatsDump());
+
+//		String stats = toNotifyString();
+//		try {
+//			Stats.writeReport(RUNTIME_STATS_FILENAME,stats);
+//		}
+//		catch (IOException ioe) {
+//			System.err.println(ioe);
+//			ioe.printStackTrace(System.err);
+//		}
+	}
+
+	public void setParser(DebugParser parser) {
+		this.parser = parser;
+	}
+
+	// R E P O R T I N G
+
+	public String toNotifyString() {
+		StringBuffer buf = new StringBuffer();
+		buf.append(Version);
+		buf.append('\t');
+		buf.append(parser.getClass().getName());
+//		buf.append('\t');
+//		buf.append(numRuleInvocations);
+//		buf.append('\t');
+//		buf.append(maxRuleInvocationDepth);
+//		buf.append('\t');
+//		buf.append(numFixedDecisions);
+//		buf.append('\t');
+//		buf.append(Stats.min(decisionMaxFixedLookaheads));
+//		buf.append('\t');
+//		buf.append(Stats.max(decisionMaxFixedLookaheads));
+//		buf.append('\t');
+//		buf.append(Stats.avg(decisionMaxFixedLookaheads));
+//		buf.append('\t');
+//		buf.append(Stats.stddev(decisionMaxFixedLookaheads));
+//		buf.append('\t');
+//		buf.append(numCyclicDecisions);
+//		buf.append('\t');
+//		buf.append(Stats.min(decisionMaxCyclicLookaheads));
+//		buf.append('\t');
+//		buf.append(Stats.max(decisionMaxCyclicLookaheads));
+//		buf.append('\t');
+//		buf.append(Stats.avg(decisionMaxCyclicLookaheads));
+//		buf.append('\t');
+//		buf.append(Stats.stddev(decisionMaxCyclicLookaheads));
+//		buf.append('\t');
+//		buf.append(numBacktrackDecisions);
+//		buf.append('\t');
+//		buf.append(Stats.min(toArray(decisionMaxSynPredLookaheads)));
+//		buf.append('\t');
+//		buf.append(Stats.max(toArray(decisionMaxSynPredLookaheads)));
+//		buf.append('\t');
+//		buf.append(Stats.avg(toArray(decisionMaxSynPredLookaheads)));
+//		buf.append('\t');
+//		buf.append(Stats.stddev(toArray(decisionMaxSynPredLookaheads)));
+//		buf.append('\t');
+//		buf.append(numSemanticPredicates);
+//		buf.append('\t');
+//		buf.append(parser.getTokenStream().size());
+//		buf.append('\t');
+//		buf.append(numHiddenTokens);
+//		buf.append('\t');
+//		buf.append(numCharsMatched);
+//		buf.append('\t');
+//		buf.append(numHiddenCharsMatched);
+//		buf.append('\t');
+//		buf.append(numberReportedErrors);
+//		buf.append('\t');
+//		buf.append(numMemoizationCacheHits);
+//		buf.append('\t');
+//		buf.append(numMemoizationCacheMisses);
+//		buf.append('\t');
+//		buf.append(numGuessingRuleInvocations);
+//		buf.append('\t');
+//		buf.append(numMemoizationCacheEntries);
+		return buf.toString();
+	}
+
+	public String toString() {
+		return toString(getReport());
+	}
+
+	public ProfileStats getReport() {
+//		TokenStream input = parser.getTokenStream();
+//		for (int i=0; i<input.size()&& lastRealTokenTouchedInDecision !=null&&i<= lastRealTokenTouchedInDecision.getTokenIndex(); i++) {
+//			Token t = input.get(i);
+//			if ( t.getChannel()!=Token.DEFAULT_CHANNEL ) {
+//				stats.numHiddenTokens++;
+//				stats.numHiddenCharsMatched += t.getText().length();
+//			}
+//		}
+		stats.Version = Version;
+		stats.name = parser.getClass().getName();
+		stats.numUniqueRulesInvoked = uniqueRules.size();
+		//stats.numCharsMatched = lastTokenConsumed.getStopIndex() + 1;
+		return stats;
+	}
+
+	public DoubleKeyMap getDecisionStats() {
+		return decisions;
+	}
+
+	public List getDecisionEvents() {
+		return decisionEvents;
+	}
+
+	public static String toString(ProfileStats stats) {
+		StringBuffer buf = new StringBuffer();
+		buf.append("ANTLR Runtime Report; Profile Version ");
+		buf.append(stats.Version);
+		buf.append(newline);
+		buf.append("parser name ");
+		buf.append(stats.name);
+		buf.append(newline);
+		buf.append("Number of rule invocations ");
+		buf.append(stats.numRuleInvocations);
+		buf.append(newline);
+		buf.append("Number of unique rules visited ");
+		buf.append(stats.numUniqueRulesInvoked);
+		buf.append(newline);
+		buf.append("Number of decision events ");
+		buf.append(stats.numDecisionEvents);
+		buf.append(newline);
+		buf.append("Overall average k per decision event ");
+		buf.append(stats.avgkPerDecisionEvent);
+		buf.append(newline);
+		buf.append("Number of backtracking occurrences (can be multiple per decision) ");
+		buf.append(stats.numBacktrackOccurrences);
+		buf.append(newline);
+		buf.append("Overall average k per decision event that backtracks ");
+		buf.append(stats.avgkPerBacktrackingDecisionEvent);
+		buf.append(newline);
+		buf.append("Number of rule invocations while backtracking ");
+		buf.append(stats.numGuessingRuleInvocations);
+		buf.append(newline);
+		buf.append("num decisions that potentially backtrack ");
+		buf.append(stats.numDecisionsThatPotentiallyBacktrack);
+		buf.append(newline);
+		buf.append("num decisions that do backtrack ");
+		buf.append(stats.numDecisionsThatDoBacktrack);
+		buf.append(newline);
+		buf.append("num decisions that potentially backtrack but don't ");
+		buf.append(stats.numDecisionsThatPotentiallyBacktrack - stats.numDecisionsThatDoBacktrack);
+		buf.append(newline);
+		buf.append("average % of time a potentially backtracking decision backtracks ");
+		buf.append(stats.averageDecisionPercentBacktracks);
+		buf.append(newline);
+		buf.append("num unique decisions covered ");
+		buf.append(stats.numDecisionsCovered);
+		buf.append(newline);
+		buf.append("max rule invocation nesting depth ");
+		buf.append(stats.maxRuleInvocationDepth);
+		buf.append(newline);
+
+//		buf.append("number of fixed lookahead decisions ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("min lookahead used in a fixed lookahead decision ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("max lookahead used in a fixed lookahead decision ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("average lookahead depth used in fixed lookahead decisions ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("standard deviation of depth used in fixed lookahead decisions ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("number of arbitrary lookahead decisions ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("min lookahead used in an arbitrary lookahead decision ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("max lookahead used in an arbitrary lookahead decision ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("average lookahead depth used in arbitrary lookahead decisions ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("standard deviation of depth used in arbitrary lookahead decisions ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("number of evaluated syntactic predicates ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("min lookahead used in a syntactic predicate ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("max lookahead used in a syntactic predicate ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("average lookahead depth used in syntactic predicates ");
+//		buf.append();
+//		buf.append('\n');
+//		buf.append("standard deviation of depth used in syntactic predicates ");
+//		buf.append();
+//		buf.append('\n');
+		buf.append("rule memoization cache size ");
+		buf.append(stats.numMemoizationCacheEntries);
+		buf.append(newline);
+		buf.append("number of rule memoization cache hits ");
+		buf.append(stats.numMemoizationCacheHits);
+		buf.append(newline);
+		buf.append("number of rule memoization cache misses ");
+		buf.append(stats.numMemoizationCacheMisses);
+		buf.append(newline);
+//		buf.append("number of evaluated semantic predicates ");
+//		buf.append();
+//		buf.append(newline);
+		buf.append("number of tokens ");
+		buf.append(stats.numTokens);
+		buf.append(newline);
+		buf.append("number of hidden tokens ");
+		buf.append(stats.numHiddenTokens);
+		buf.append(newline);
+		buf.append("number of char ");
+		buf.append(stats.numCharsMatched);
+		buf.append(newline);
+		buf.append("number of hidden char ");
+		buf.append(stats.numHiddenCharsMatched);
+		buf.append(newline);
+		buf.append("number of syntax errors ");
+		buf.append(stats.numReportedErrors);
+		buf.append(newline);
+		return buf.toString();
+	}
+
+	public String getDecisionStatsDump() {
+		StringBuffer buf = new StringBuffer();
+		buf.append("location");
+		buf.append(DATA_SEP);
+		buf.append("n");
+		buf.append(DATA_SEP);
+		buf.append("avgk");
+		buf.append(DATA_SEP);
+		buf.append("maxk");
+		buf.append(DATA_SEP);
+		buf.append("synpred");
+		buf.append(DATA_SEP);
+		buf.append("sempred");
+		buf.append(DATA_SEP);
+		buf.append("canbacktrack");
+		buf.append("\n");
+		for (String fileName : decisions.keySet()) {
+			for (int d : decisions.keySet(fileName)) {
+				DecisionDescriptor s = decisions.get(fileName, d);
+				buf.append(s.decision);
+				buf.append("@");
+				buf.append(locationDescription(s.fileName,s.ruleName,s.line,s.pos)); // decision number
+				buf.append(DATA_SEP);
+				buf.append(s.n);
+				buf.append(DATA_SEP);
+				buf.append(String.format("%.2f",s.avgk));
+				buf.append(DATA_SEP);
+				buf.append(s.maxk);
+				buf.append(DATA_SEP);
+				buf.append(s.numBacktrackOccurrences);
+				buf.append(DATA_SEP);
+				buf.append(s.numSemPredEvals);
+				buf.append(DATA_SEP);
+				buf.append(s.couldBacktrack ?"1":"0");
+				buf.append(newline);
+			}
+		}
+		return buf.toString();
+	}
+
+	protected int[] trim(int[] X, int n) {
+		if ( n<X.length ) {
+			int[] trimmed = new int[n];
+			System.arraycopy(X,0,trimmed,0,n);
+			X = trimmed;
+		}
+		return X;
+	}
+
+	protected int[] toArray(List a) {
+		int[] x = new int[a.size()];
+		for (int i = 0; i < a.size(); i++) {
+			Integer I = (Integer) a.get(i);
+			x[i] = I.intValue();
+		}
+		return x;
+	}
+
+	/** Get num hidden tokens between i..j inclusive */
+	public int getNumberOfHiddenTokens(int i, int j) {
+		int n = 0;
+		TokenStream input = parser.getTokenStream();
+		for (int ti = i; ti<input.size() && ti <= j; ti++) {
+			Token t = input.get(ti);
+			if ( t.getChannel()!=Token.DEFAULT_CHANNEL ) {
+				n++;
+			}
+		}
+		return n;
+	}
+
+	protected String locationDescription() {
+		return locationDescription(
+			currentGrammarFileName.peek(),
+			currentRuleName.peek(),
+			currentLine.peek(),
+			currentPos.peek());
+	}
+
+	protected String locationDescription(String file, String rule, int line, int pos) {
+		return file+":"+line+":"+pos+"(" + rule + ")";
+	}
+}
diff --git a/src/org/antlr/runtime/debug/RemoteDebugEventSocketListener.java b/src/org/antlr/runtime/debug/RemoteDebugEventSocketListener.java
new file mode 100755
index 0000000..933fdae
--- /dev/null
+++ b/src/org/antlr/runtime/debug/RemoteDebugEventSocketListener.java
@@ -0,0 +1,527 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.CharStream;
+import org.antlr.runtime.tree.BaseTree;
+import org.antlr.runtime.tree.Tree;
+
+import java.io.*;
+import java.net.ConnectException;
+import java.net.Socket;
+import java.util.StringTokenizer;
+
+public class RemoteDebugEventSocketListener implements Runnable {
+	static final int MAX_EVENT_ELEMENTS = 8;
+	DebugEventListener listener;
+	String machine;
+	int port;
+	Socket channel = null;
+	PrintWriter out;
+	BufferedReader in;
+	String event;
+	/** Version of ANTLR (dictates events) */
+	public String version;
+	public String grammarFileName;
+	/** Track the last token index we saw during a consume.  If same, then
+	 *  set a flag that we have a problem.
+	 */
+	int previousTokenIndex = -1;
+	boolean tokenIndexesInvalid = false;
+
+	public static class ProxyToken implements Token {
+		int index;
+		int type;
+		int channel;
+		int line;
+		int charPos;
+		String text;
+		public ProxyToken(int index) { this.index = index; }		
+		public ProxyToken(int index, int type, int channel,
+						  int line, int charPos, String text)
+		{
+			this.index = index;
+			this.type = type;
+			this.channel = channel;
+			this.line = line;
+			this.charPos = charPos;
+			this.text = text;
+		}
+		public String getText() {
+			return text;
+		}
+		public void setText(String text) {
+			this.text = text;
+		}
+		public int getType() {
+			return type;
+		}
+		public void setType(int ttype) {
+			this.type = ttype;
+		}
+		public int getLine() {
+			return line;
+		}
+		public void setLine(int line) {
+			this.line = line;
+		}
+		public int getCharPositionInLine() {
+			return charPos;
+		}
+		public void setCharPositionInLine(int pos) {
+			this.charPos = pos;
+		}
+		public int getChannel() {
+			return channel;
+		}
+		public void setChannel(int channel) {
+			this.channel = channel;
+		}
+		public int getTokenIndex() {
+			return index;
+		}
+		public void setTokenIndex(int index) {
+			this.index = index;
+		}
+		public CharStream getInputStream() {
+			return null;
+		}
+		public void setInputStream(CharStream input) {
+		}
+		public String toString() {
+			String channelStr = "";
+			if ( channel!=Token.DEFAULT_CHANNEL ) {
+				channelStr=",channel="+channel;
+			}
+			return "["+getText()+"/<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+",@"+index+"]";
+		}
+	}
+
+	public static class ProxyTree extends BaseTree {
+		public int ID;
+		public int type;
+		public int line = 0;
+		public int charPos = -1;
+		public int tokenIndex = -1;
+		public String text;
+		
+		public ProxyTree(int ID, int type, int line, int charPos, int tokenIndex, String text) {
+			this.ID = ID;
+			this.type = type;
+			this.line = line;
+			this.charPos = charPos;
+			this.tokenIndex = tokenIndex;
+			this.text = text;
+		}
+
+		public ProxyTree(int ID) { this.ID = ID; }
+
+		public int getTokenStartIndex() { return tokenIndex; }
+		public void setTokenStartIndex(int index) {	}
+		public int getTokenStopIndex() { return 0; }
+		public void setTokenStopIndex(int index) { }
+		public Tree dupNode() {	return null; }
+		public int getType() { return type; }
+		public String getText() { return text; }
+		public String toString() {
+			return "fix this";
+		}
+	}
+
+	public RemoteDebugEventSocketListener(DebugEventListener listener,
+										  String machine,
+										  int port) throws IOException
+	{
+		this.listener = listener;
+		this.machine = machine;
+		this.port = port;
+
+        if( !openConnection() ) {
+            throw new ConnectException();
+        }
+	}
+
+	protected void eventHandler() {
+		try {
+			handshake();
+			event = in.readLine();
+			while ( event!=null ) {
+				dispatch(event);
+				ack();
+				event = in.readLine();
+			}
+		}
+		catch (Exception e) {
+			System.err.println(e);
+			e.printStackTrace(System.err);
+		}
+		finally {
+            closeConnection();
+		}
+	}
+
+    protected boolean openConnection() {
+        boolean success = false;
+        try {
+            channel = new Socket(machine, port);
+            channel.setTcpNoDelay(true);
+			OutputStream os = channel.getOutputStream();
+			OutputStreamWriter osw = new OutputStreamWriter(os, "UTF8");
+			out = new PrintWriter(new BufferedWriter(osw));
+			InputStream is = channel.getInputStream();
+			InputStreamReader isr = new InputStreamReader(is, "UTF8");
+			in = new BufferedReader(isr);
+            success = true;
+        } catch(Exception e) {
+            System.err.println(e);
+        }
+        return success;
+    }
+
+    protected void closeConnection() {
+        try {
+            in.close(); in = null;
+            out.close(); out = null;
+            channel.close(); channel=null;
+        }
+        catch (Exception e) {
+            System.err.println(e);
+            e.printStackTrace(System.err);
+        }
+        finally {
+            if ( in!=null ) {
+                try {in.close();} catch (IOException ioe) {
+                    System.err.println(ioe);
+                }
+            }
+            if ( out!=null ) {
+                out.close();
+            }
+            if ( channel!=null ) {
+                try {channel.close();} catch (IOException ioe) {
+                    System.err.println(ioe);
+                }
+            }
+        }
+
+    }
+
+	protected void handshake() throws IOException {
+		String antlrLine = in.readLine();
+		String[] antlrElements = getEventElements(antlrLine);
+		version = antlrElements[1];
+		String grammarLine = in.readLine();
+		String[] grammarElements = getEventElements(grammarLine);
+		grammarFileName = grammarElements[1];
+		ack();
+		listener.commence(); // inform listener after handshake
+	}
+
+	protected void ack() {
+        out.println("ack");
+		out.flush();
+	}
+
+	protected void dispatch(String line) {
+        //System.out.println("event: "+line);
+        String[] elements = getEventElements(line);
+		if ( elements==null || elements[0]==null ) {
+			System.err.println("unknown debug event: "+line);
+			return;
+		}
+		if ( elements[0].equals("enterRule") ) {
+			listener.enterRule(elements[1], elements[2]);
+		}
+		else if ( elements[0].equals("exitRule") ) {
+			listener.exitRule(elements[1], elements[2]);
+		}
+		else if ( elements[0].equals("enterAlt") ) {
+			listener.enterAlt(Integer.parseInt(elements[1]));
+		}
+		else if ( elements[0].equals("enterSubRule") ) {
+			listener.enterSubRule(Integer.parseInt(elements[1]));
+		}
+		else if ( elements[0].equals("exitSubRule") ) {
+			listener.exitSubRule(Integer.parseInt(elements[1]));
+		}
+		else if ( elements[0].equals("enterDecision") ) {
+			listener.enterDecision(Integer.parseInt(elements[1]), elements[2].equals("true"));
+		}
+		else if ( elements[0].equals("exitDecision") ) {
+			listener.exitDecision(Integer.parseInt(elements[1]));
+		}
+		else if ( elements[0].equals("location") ) {
+			listener.location(Integer.parseInt(elements[1]),
+							  Integer.parseInt(elements[2]));
+		}
+		else if ( elements[0].equals("consumeToken") ) {
+			ProxyToken t = deserializeToken(elements, 1);
+			if ( t.getTokenIndex() == previousTokenIndex ) {
+				tokenIndexesInvalid = true;
+			}
+			previousTokenIndex = t.getTokenIndex();
+			listener.consumeToken(t);
+		}
+		else if ( elements[0].equals("consumeHiddenToken") ) {
+			ProxyToken t = deserializeToken(elements, 1);
+			if ( t.getTokenIndex() == previousTokenIndex ) {
+				tokenIndexesInvalid = true;
+			}
+			previousTokenIndex = t.getTokenIndex();
+			listener.consumeHiddenToken(t);
+		}
+		else if ( elements[0].equals("LT") ) {
+			Token t = deserializeToken(elements, 2);
+			listener.LT(Integer.parseInt(elements[1]), t);
+		}
+		else if ( elements[0].equals("mark") ) {
+			listener.mark(Integer.parseInt(elements[1]));
+		}
+		else if ( elements[0].equals("rewind") ) {
+			if ( elements[1]!=null ) {
+				listener.rewind(Integer.parseInt(elements[1]));
+			}
+			else {
+				listener.rewind();
+			}
+		}
+		else if ( elements[0].equals("beginBacktrack") ) {
+			listener.beginBacktrack(Integer.parseInt(elements[1]));
+		}
+		else if ( elements[0].equals("endBacktrack") ) {
+			int level = Integer.parseInt(elements[1]);
+			int successI = Integer.parseInt(elements[2]);
+			listener.endBacktrack(level, successI==DebugEventListener.TRUE);
+		}
+		else if ( elements[0].equals("exception") ) {
+			String excName = elements[1];
+			String indexS = elements[2];
+			String lineS = elements[3];
+			String posS = elements[4];
+			Class excClass = null;
+			try {
+				excClass = Class.forName(excName);
+				RecognitionException e =
+					(RecognitionException)excClass.newInstance();
+				e.index = Integer.parseInt(indexS);
+				e.line = Integer.parseInt(lineS);
+				e.charPositionInLine = Integer.parseInt(posS);
+				listener.recognitionException(e);
+			}
+			catch (ClassNotFoundException cnfe) {
+				System.err.println("can't find class "+cnfe);
+				cnfe.printStackTrace(System.err);
+			}
+			catch (InstantiationException ie) {
+				System.err.println("can't instantiate class "+ie);
+				ie.printStackTrace(System.err);
+			}
+			catch (IllegalAccessException iae) {
+				System.err.println("can't access class "+iae);
+				iae.printStackTrace(System.err);
+			}
+		}
+		else if ( elements[0].equals("beginResync") ) {
+			listener.beginResync();
+		}
+		else if ( elements[0].equals("endResync") ) {
+			listener.endResync();
+		}
+		else if ( elements[0].equals("terminate") ) {
+			listener.terminate();
+		}
+		else if ( elements[0].equals("semanticPredicate") ) {
+			Boolean result = Boolean.valueOf(elements[1]);
+			String predicateText = elements[2];
+			predicateText = unEscapeNewlines(predicateText);
+			listener.semanticPredicate(result.booleanValue(),
+									   predicateText);
+		}
+		else if ( elements[0].equals("consumeNode") ) {
+			ProxyTree node = deserializeNode(elements, 1);
+			listener.consumeNode(node);
+		}
+		else if ( elements[0].equals("LN") ) {
+			int i = Integer.parseInt(elements[1]);
+			ProxyTree node = deserializeNode(elements, 2);
+			listener.LT(i, node);
+		}
+		else if ( elements[0].equals("createNodeFromTokenElements") ) {
+			int ID = Integer.parseInt(elements[1]);
+			int type = Integer.parseInt(elements[2]);
+			String text = elements[3];
+			text = unEscapeNewlines(text);
+			ProxyTree node = new ProxyTree(ID, type, -1, -1, -1, text);
+			listener.createNode(node);
+		}
+		else if ( elements[0].equals("createNode") ) {
+			int ID = Integer.parseInt(elements[1]);
+			int tokenIndex = Integer.parseInt(elements[2]);
+			// create dummy node/token filled with ID, tokenIndex
+			ProxyTree node = new ProxyTree(ID);
+			ProxyToken token = new ProxyToken(tokenIndex);
+			listener.createNode(node, token);
+		}
+		else if ( elements[0].equals("nilNode") ) {
+			int ID = Integer.parseInt(elements[1]);
+			ProxyTree node = new ProxyTree(ID);
+			listener.nilNode(node);
+		}
+		else if ( elements[0].equals("errorNode") ) {
+			// TODO: do we need a special tree here?
+			int ID = Integer.parseInt(elements[1]);
+			int type = Integer.parseInt(elements[2]);
+			String text = elements[3];
+			text = unEscapeNewlines(text);
+			ProxyTree node = new ProxyTree(ID, type, -1, -1, -1, text);
+			listener.errorNode(node);
+		}
+		else if ( elements[0].equals("becomeRoot") ) {
+			int newRootID = Integer.parseInt(elements[1]);
+			int oldRootID = Integer.parseInt(elements[2]);
+			ProxyTree newRoot = new ProxyTree(newRootID);
+			ProxyTree oldRoot = new ProxyTree(oldRootID);
+			listener.becomeRoot(newRoot, oldRoot);
+		}
+		else if ( elements[0].equals("addChild") ) {
+			int rootID = Integer.parseInt(elements[1]);
+			int childID = Integer.parseInt(elements[2]);
+			ProxyTree root = new ProxyTree(rootID);
+			ProxyTree child = new ProxyTree(childID);
+			listener.addChild(root, child);
+		}
+		else if ( elements[0].equals("setTokenBoundaries") ) {
+			int ID = Integer.parseInt(elements[1]);
+			ProxyTree node = new ProxyTree(ID);
+			listener.setTokenBoundaries(
+				node,
+				Integer.parseInt(elements[2]),
+				Integer.parseInt(elements[3]));
+		}
+		else {
+			System.err.println("unknown debug event: "+line);
+		}
+	}
+
+	protected ProxyTree deserializeNode(String[] elements, int offset) {
+		int ID = Integer.parseInt(elements[offset+0]);
+		int type = Integer.parseInt(elements[offset+1]);
+		int tokenLine = Integer.parseInt(elements[offset+2]);
+		int charPositionInLine = Integer.parseInt(elements[offset+3]);
+		int tokenIndex = Integer.parseInt(elements[offset+4]);
+		String text = elements[offset+5];
+		text = unEscapeNewlines(text);
+		return new ProxyTree(ID, type, tokenLine, charPositionInLine, tokenIndex, text);
+	}
+
+	protected ProxyToken deserializeToken(String[] elements,
+										  int offset)
+	{
+		String indexS = elements[offset+0];
+		String typeS = elements[offset+1];
+		String channelS = elements[offset+2];
+		String lineS = elements[offset+3];
+		String posS = elements[offset+4];
+		String text = elements[offset+5];
+		text = unEscapeNewlines(text);
+		int index = Integer.parseInt(indexS);
+		ProxyToken t =
+			new ProxyToken(index,
+						   Integer.parseInt(typeS),
+						   Integer.parseInt(channelS),
+						   Integer.parseInt(lineS),
+						   Integer.parseInt(posS),
+						   text);
+		return t;
+	}
+
+	/** Create a thread to listen to the remote running recognizer */
+	public void start() {
+		Thread t = new Thread(this);
+		t.start();
+	}
+
+	public void run() {
+		eventHandler();
+	}
+
+	// M i s c
+
+	public String[] getEventElements(String event) {
+		if ( event==null ) {
+			return null;
+		}
+		String[] elements = new String[MAX_EVENT_ELEMENTS];
+		String str = null; // a string element if present (must be last)
+		try {
+			int firstQuoteIndex = event.indexOf('"');
+			if ( firstQuoteIndex>=0 ) {
+				// treat specially; has a string argument like "a comment\n
+				// Note that the string is terminated by \n not end quote.
+				// Easier to parse that way.
+				String eventWithoutString = event.substring(0,firstQuoteIndex);
+				str = event.substring(firstQuoteIndex+1,event.length());
+				event = eventWithoutString;
+			}
+			StringTokenizer st = new StringTokenizer(event, "\t", false);
+			int i = 0;
+			while ( st.hasMoreTokens() ) {
+				if ( i>=MAX_EVENT_ELEMENTS ) {
+					// ErrorManager.internalError("event has more than "+MAX_EVENT_ELEMENTS+" args: "+event);
+					return elements;
+				}
+				elements[i] = st.nextToken();
+				i++;
+			}
+			if ( str!=null ) {
+				elements[i] = str;
+			}
+		}
+		catch (Exception e) {
+			e.printStackTrace(System.err);
+		}
+		return elements;
+	}
+
+	protected String unEscapeNewlines(String txt) {
+		// this unescape is slow but easy to understand
+		txt = txt.replaceAll("%0A","\n");  // unescape \n
+		txt = txt.replaceAll("%0D","\r");  // unescape \r
+		txt = txt.replaceAll("%25","%");   // undo escaped escape chars
+		return txt;
+	}
+
+	public boolean tokenIndexesAreInvalid() {
+		return false;
+		//return tokenIndexesInvalid;
+	}
+
+}
+
diff --git a/src/org/antlr/runtime/debug/TraceDebugEventListener.java b/src/org/antlr/runtime/debug/TraceDebugEventListener.java
new file mode 100755
index 0000000..de9366d
--- /dev/null
+++ b/src/org/antlr/runtime/debug/TraceDebugEventListener.java
@@ -0,0 +1,96 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.tree.TreeAdaptor;
+
+/** Print out (most of) the events... Useful for debugging, testing... */
+public class TraceDebugEventListener extends BlankDebugEventListener {
+	TreeAdaptor adaptor;
+
+	public TraceDebugEventListener(TreeAdaptor adaptor) {
+		this.adaptor = adaptor;
+	}
+
+	public void enterRule(String ruleName) { System.out.println("enterRule "+ruleName); }
+	public void exitRule(String ruleName) { System.out.println("exitRule "+ruleName); }
+	public void enterSubRule(int decisionNumber) { System.out.println("enterSubRule"); }
+	public void exitSubRule(int decisionNumber) { System.out.println("exitSubRule"); }
+	public void location(int line, int pos) {System.out.println("location "+line+":"+pos);}
+
+	// Tree parsing stuff
+
+	public void consumeNode(Object t) {
+		int ID = adaptor.getUniqueID(t);
+		String text = adaptor.getText(t);
+		int type = adaptor.getType(t);
+		System.out.println("consumeNode "+ID+" "+text+" "+type);
+	}
+
+	public void LT(int i, Object t) {
+		int ID = adaptor.getUniqueID(t);
+		String text = adaptor.getText(t);
+		int type = adaptor.getType(t);
+		System.out.println("LT "+i+" "+ID+" "+text+" "+type);
+	}
+
+
+	// AST stuff
+	public void nilNode(Object t) {System.out.println("nilNode "+adaptor.getUniqueID(t));}
+
+	public void createNode(Object t) {
+		int ID = adaptor.getUniqueID(t);
+		String text = adaptor.getText(t);
+		int type = adaptor.getType(t);
+		System.out.println("create "+ID+": "+text+", "+type);
+	}
+
+	public void createNode(Object node, Token token) {
+		int ID = adaptor.getUniqueID(node);
+		String text = adaptor.getText(node);
+		int tokenIndex = token.getTokenIndex();
+		System.out.println("create "+ID+": "+tokenIndex);
+	}
+
+	public void becomeRoot(Object newRoot, Object oldRoot) {
+		System.out.println("becomeRoot "+adaptor.getUniqueID(newRoot)+", "+
+						   adaptor.getUniqueID(oldRoot));
+	}
+
+	public void addChild(Object root, Object child) {
+		System.out.println("addChild "+adaptor.getUniqueID(root)+", "+
+						   adaptor.getUniqueID(child));
+	}
+
+	public void setTokenBoundaries(Object t, int tokenStartIndex, int tokenStopIndex) {
+		System.out.println("setTokenBoundaries "+adaptor.getUniqueID(t)+", "+
+						   tokenStartIndex+", "+tokenStopIndex);
+	}
+}
+
diff --git a/src/org/antlr/runtime/debug/Tracer.java b/src/org/antlr/runtime/debug/Tracer.java
new file mode 100755
index 0000000..c2c73da
--- /dev/null
+++ b/src/org/antlr/runtime/debug/Tracer.java
@@ -0,0 +1,65 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.debug;
+
+import org.antlr.runtime.IntStream;
+import org.antlr.runtime.TokenStream;
+
+/** The default tracer mimics the traceParser behavior of ANTLR 2.x.
+ *  This listens for debugging events from the parser and implies
+ *  that you cannot debug and trace at the same time.
+ */
+public class Tracer extends BlankDebugEventListener {
+	public IntStream input;
+	protected int level = 0;
+
+	public Tracer(IntStream input) {
+		this.input = input;
+	}
+
+	public void enterRule(String ruleName) {
+		for (int i=1; i<=level; i++) {System.out.print(" ");}
+		System.out.println("> "+ruleName+" lookahead(1)="+getInputSymbol(1));
+		level++;
+	}
+
+	public void exitRule(String ruleName) {
+		level--;
+		for (int i=1; i<=level; i++) {System.out.print(" ");}
+		System.out.println("< "+ruleName+" lookahead(1)="+getInputSymbol(1));
+	}
+
+	public Object getInputSymbol(int k) {
+		if ( input instanceof TokenStream ) {
+			return ((TokenStream)input).LT(k);
+		}
+		return new Character((char)input.LA(k));
+	}
+}
+
+
diff --git a/src/org/antlr/runtime/misc/DoubleKeyMap.java b/src/org/antlr/runtime/misc/DoubleKeyMap.java
new file mode 100755
index 0000000..b69ae32
--- /dev/null
+++ b/src/org/antlr/runtime/misc/DoubleKeyMap.java
@@ -0,0 +1,62 @@
+package org.antlr.runtime.misc;
+
+import java.util.*;
+
+/** Sometimes we need to map a key to a value but key is two pieces of data.
+ *  This nested hash table saves creating a single key each time we access
+ *  map; avoids mem creation.
+ */
+public class DoubleKeyMap<Key1, Key2, Value> {
+	Map<Key1, Map<Key2, Value>> data = new LinkedHashMap<Key1, Map<Key2, Value>>();
+
+	public Value put(Key1 k1, Key2 k2, Value v) {
+		Map<Key2, Value> data2 = data.get(k1);
+		Value prev = null;
+		if ( data2==null ) {
+			data2 = new LinkedHashMap<Key2, Value>();
+			data.put(k1, data2);
+		}
+		else {
+			prev = data2.get(k2);
+		}
+		data2.put(k2, v);
+		return prev;
+	}
+
+	public Value get(Key1 k1, Key2 k2) {
+		Map<Key2, Value> data2 = data.get(k1);
+		if ( data2==null ) return null;
+		return data2.get(k2);
+	}
+
+	public Map<Key2, Value> get(Key1 k1) { return data.get(k1); }
+
+	/** Get all values associated with primary key */
+	public Collection<Value> values(Key1 k1) {
+		Map<Key2, Value> data2 = data.get(k1);
+		if ( data2==null ) return null;
+		return data2.values();
+	}
+
+	/** get all primary keys */
+	public Set<Key1> keySet() {
+		return data.keySet();
+	}
+
+	/** get all secondary keys associated with a primary key */
+	public Set<Key2> keySet(Key1 k1) {
+		Map<Key2, Value> data2 = data.get(k1);
+		if ( data2==null ) return null;
+		return data2.keySet();
+	}
+
+	public Collection<Value> values() {
+		Set<Value> s = new HashSet<Value>();
+		for (Map<Key2, Value> k2 : data.values()) {
+			for (Value v : k2.values()) {
+				s.add(v);
+			}
+		}
+		return s;
+	}
+}
diff --git a/src/org/antlr/runtime/misc/FastQueue.java b/src/org/antlr/runtime/misc/FastQueue.java
new file mode 100755
index 0000000..08843dd
--- /dev/null
+++ b/src/org/antlr/runtime/misc/FastQueue.java
@@ -0,0 +1,100 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.misc;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.NoSuchElementException;
+
+/** A queue that can dequeue and get(i) in O(1) and grow arbitrarily large.
+ *  A linked list is fast at dequeue but slow at get(i).  An array is
+ *  the reverse.  This is O(1) for both operations.
+ *
+ *  List grows until you dequeue last element at end of buffer. Then
+ *  it resets to start filling at 0 again.  If adds/removes are balanced, the
+ *  buffer will not grow too large.
+ *
+ *  No iterator stuff as that's not how we'll use it.
+ */
+public class FastQueue<T> {
+    /** dynamically-sized buffer of elements */
+    protected List<T> data = new ArrayList<T>();
+    /** index of next element to fill */
+    protected int p = 0;
+	protected int range = -1; // how deep have we gone?	
+
+    public void reset() { clear(); }
+    public void clear() { p = 0; data.clear(); }
+
+    /** Get and remove first element in queue */
+    public T remove() {
+        T o = elementAt(0);
+        p++;
+        // have we hit end of buffer?
+        if ( p == data.size() ) {
+            // if so, it's an opportunity to start filling at index 0 again
+            clear(); // size goes to 0, but retains memory
+        }
+        return o;
+    }
+
+    public void add(T o) { data.add(o); }
+
+    public int size() { return data.size() - p; }
+
+	public int range() { return range; }
+
+    public T head() { return elementAt(0); }
+
+    /** Return element i elements ahead of current element.  i==0 gets
+     *  current element.  This is not an absolute index into the data list
+     *  since p defines the start of the real list.
+     */
+    public T elementAt(int i) {
+		int absIndex = p + i;
+		if ( absIndex >= data.size() ) {
+            throw new NoSuchElementException("queue index "+ absIndex +" > last index "+(data.size()-1));
+        }
+        if ( absIndex < 0 ) {
+            throw new NoSuchElementException("queue index "+ absIndex +" < 0");
+        }
+		if ( absIndex>range ) range = absIndex;
+        return data.get(absIndex);
+    }
+
+    /** Return string of current buffer contents; non-destructive */
+    public String toString() {
+        StringBuffer buf = new StringBuffer();
+        int n = size();
+        for (int i=0; i<n; i++) {
+            buf.append(elementAt(i));
+            if ( (i+1)<n ) buf.append(" ");
+        }
+        return buf.toString();
+    }
+}
\ No newline at end of file
diff --git a/src/org/antlr/runtime/misc/IntArray.java b/src/org/antlr/runtime/misc/IntArray.java
new file mode 100755
index 0000000..bc484aa
--- /dev/null
+++ b/src/org/antlr/runtime/misc/IntArray.java
@@ -0,0 +1,87 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.misc;
+
+/** A dynamic array that uses int not Integer objects. In principle this
+ *  is more efficient in time, but certainly in space.
+ *
+ *  This is simple enough that you can access the data array directly,
+ *  but make sure that you append elements only with add() so that you
+ *  get dynamic sizing.  Make sure to call ensureCapacity() when you are
+ *  manually adding new elements.
+ *
+ *  Doesn't impl List because it doesn't return objects and I mean this
+ *  really as just an array not a List per se.  Manipulate the elements
+ *  at will.  This has stack methods too.
+ *
+ *  When runtime can be 1.5, I'll make this generic.
+ */
+public class IntArray {
+	public static final int INITIAL_SIZE = 10;
+	public int[] data;
+	protected int p = -1;
+
+	public void add(int v) {
+		ensureCapacity(p+1);
+		data[++p] = v;
+	}
+
+	public void push(int v) {
+		add(v);
+	}
+
+	public int pop() {
+		int v = data[p];
+		p--;
+		return v;
+	}
+
+	/** This only tracks elements added via push/add. */
+	public int size() {
+		return p;
+	}
+
+    public void clear() {
+        p = -1;
+    }
+
+    public void ensureCapacity(int index) {
+		if ( data==null ) {
+			data = new int[INITIAL_SIZE];
+		}
+		else if ( (index+1)>=data.length ) {
+			int newSize = data.length*2;
+			if ( index>newSize ) {
+				newSize = index+1;
+			}
+			int[] newData = new int[newSize];
+			System.arraycopy(data, 0, newData, 0, data.length);
+			data = newData;
+		}
+	}
+}
diff --git a/src/org/antlr/runtime/misc/LookaheadStream.java b/src/org/antlr/runtime/misc/LookaheadStream.java
new file mode 100755
index 0000000..6f19c44
--- /dev/null
+++ b/src/org/antlr/runtime/misc/LookaheadStream.java
@@ -0,0 +1,161 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.misc;
+
+import org.antlr.runtime.Token;
+
+import java.util.NoSuchElementException;
+
+/** A lookahead queue that knows how to mark/release locations
+ *  in the buffer for backtracking purposes. Any markers force the FastQueue
+ *  superclass to keep all tokens until no more markers; then can reset
+ *  to avoid growing a huge buffer.
+ */
+public abstract class LookaheadStream<T> extends FastQueue<T> {
+    public static final int UNINITIALIZED_EOF_ELEMENT_INDEX = Integer.MAX_VALUE;
+
+    /** Absolute token index. It's the index of the symbol about to be
+	 *  read via LT(1). Goes from 0 to numtokens.
+     */
+    protected int currentElementIndex = 0;
+
+    protected T prevElement;
+
+    /** Track object returned by nextElement upon end of stream;
+     *  Return it later when they ask for LT passed end of input.
+     */
+    public T eof = null;
+
+    /** Track the last mark() call result value for use in rewind(). */
+    protected int lastMarker;
+
+    /** tracks how deep mark() calls are nested */
+    protected int markDepth = 0;
+
+    public void reset() {
+        super.reset();
+        currentElementIndex = 0;
+        p = 0;
+        prevElement=null;        
+    }
+    
+    /** Implement nextElement to supply a stream of elements to this
+     *  lookahead buffer.  Return eof upon end of the stream we're pulling from.
+     */
+    public abstract T nextElement();
+
+    public abstract boolean isEOF(T o);
+
+    /** Get and remove first element in queue; override FastQueue.remove();
+     *  it's the same, just checks for backtracking.
+     */
+    public T remove() {
+        T o = elementAt(0);
+        p++;
+        // have we hit end of buffer and not backtracking?
+        if ( p == data.size() && markDepth==0 ) {
+            // if so, it's an opportunity to start filling at index 0 again
+            clear(); // size goes to 0, but retains memory
+        }
+        return o;
+    }
+
+    /** Make sure we have at least one element to remove, even if EOF */
+    public void consume() {
+        syncAhead(1);
+        prevElement = remove();
+        currentElementIndex++;
+    }
+
+    /** Make sure we have 'need' elements from current position p. Last valid
+     *  p index is data.size()-1.  p+need-1 is the data index 'need' elements
+     *  ahead.  If we need 1 element, (p+1-1)==p must be < data.size().
+     */
+    protected void syncAhead(int need) {
+        int n = (p+need-1) - data.size() + 1; // how many more elements we need?
+        if ( n > 0 ) fill(n);                 // out of elements?
+    }
+
+    /** add n elements to buffer */
+    public void fill(int n) {
+        for (int i=1; i<=n; i++) {
+            T o = nextElement();
+            if ( isEOF(o) ) eof = o;
+            data.add(o);
+        }
+    }
+
+    /** Size of entire stream is unknown; we only know buffer size from FastQueue */
+    public int size() { throw new UnsupportedOperationException("streams are of unknown size"); }
+
+    public T LT(int k) {
+		if ( k==0 ) {
+			return null;
+		}
+		if ( k<0 ) return LB(-k);
+		//System.out.print("LT(p="+p+","+k+")=");
+        syncAhead(k);
+        if ( (p+k-1) > data.size() ) return eof;
+        return elementAt(k-1);
+	}
+
+    public int index() { return currentElementIndex; }
+
+	public int mark() {
+        markDepth++;
+        lastMarker = p; // track where we are in buffer not absolute token index
+        return lastMarker;
+	}
+
+	public void release(int marker) {
+		// no resources to release
+	}
+
+	public void rewind(int marker) {
+        markDepth--;
+        seek(marker); // assume marker is top
+        // release(marker); // waste of call; it does nothing in this class
+    }
+
+	public void rewind() {
+        seek(lastMarker); // rewind but do not release marker
+    }
+
+    /** Seek to a 0-indexed position within data buffer.  Can't handle
+     *  case where you seek beyond end of existing buffer.  Normally used
+     *  to seek backwards in the buffer. Does not force loading of nodes.
+     *  Doesn't see to absolute position in input stream since this stream
+     *  is unbuffered. Seeks only into our moving window of elements.
+     */
+    public void seek(int index) { p = index; }
+
+    protected T LB(int k) {
+        if ( k==1 ) return prevElement;
+        throw new NoSuchElementException("can't look backwards more than one token in this stream");
+    }
+}
\ No newline at end of file
diff --git a/src/org/antlr/runtime/misc/Stats.java b/src/org/antlr/runtime/misc/Stats.java
new file mode 100755
index 0000000..9fdd21e
--- /dev/null
+++ b/src/org/antlr/runtime/misc/Stats.java
@@ -0,0 +1,189 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.misc;
+
+import java.io.*;
+import java.util.List;
+
+/** Stats routines needed by profiler etc...
+
+ // note that these routines return 0.0 if no values exist in the X[]
+ // which is not "correct", but it is useful so I don't generate NaN
+ // in my output
+
+ */
+public class Stats {
+	public static final String ANTLRWORKS_DIR = "antlrworks";
+
+	/** Compute the sample (unbiased estimator) standard deviation following:
+	 *
+	 *  Computing Deviations: Standard Accuracy
+	 *  Tony F. Chan and John Gregg Lewis
+	 *  Stanford University
+	 *  Communications of ACM September 1979 of Volume 22 the ACM Number 9
+	 *
+	 *  The "two-pass" method from the paper; supposed to have better
+	 *  numerical properties than the textbook summation/sqrt.  To me
+	 *  this looks like the textbook method, but I ain't no numerical
+	 *  methods guy.
+	 */
+	public static double stddev(int[] X) {
+		int m = X.length;
+		if ( m<=1 ) {
+			return 0;
+		}
+		double xbar = avg(X);
+		double s2 = 0.0;
+		for (int i=0; i<m; i++){
+			s2 += (X[i] - xbar)*(X[i] - xbar);
+		}
+		s2 = s2/(m-1);
+		return Math.sqrt(s2);
+	}
+
+	/** Compute the sample mean */
+	public static double avg(int[] X) {
+		double xbar = 0.0;
+		int m = X.length;
+		if ( m==0 ) {
+			return 0;
+		}
+		for (int i=0; i<m; i++){
+			xbar += X[i];
+		}
+		if ( xbar>=0.0 ) {
+			return xbar / m;
+		}
+		return 0.0;
+	}
+
+	public static int min(int[] X) {
+		int min = Integer.MAX_VALUE;
+		int m = X.length;
+		if ( m==0 ) {
+			return 0;
+		}
+		for (int i=0; i<m; i++){
+			if ( X[i] < min ) {
+				min = X[i];
+			}
+		}
+		return min;
+	}
+
+	public static int max(int[] X) {
+		int max = Integer.MIN_VALUE;
+		int m = X.length;
+		if ( m==0 ) {
+			return 0;
+		}
+		for (int i=0; i<m; i++){
+			if ( X[i] > max ) {
+				max = X[i];
+			}
+		}
+		return max;
+	}
+
+	/** Compute the sample mean */
+	public static double avg(List<Integer> X) {
+		double xbar = 0.0;
+		int m = X.size();
+		if ( m==0 ) {
+			return 0;
+		}
+		for (int i=0; i<m; i++){
+			xbar += X.get(i);
+		}
+		if ( xbar>=0.0 ) {
+			return xbar / m;
+		}
+		return 0.0;
+	}
+
+	public static int min(List<Integer> X) {
+		int min = Integer.MAX_VALUE;
+		int m = X.size();
+		if ( m==0 ) {
+			return 0;
+		}
+		for (int i=0; i<m; i++){
+			if ( X.get(i) < min ) {
+				min = X.get(i);
+			}
+		}
+		return min;
+	}
+
+	public static int max(List<Integer> X) {
+		int max = Integer.MIN_VALUE;
+		int m = X.size();
+		if ( m==0 ) {
+			return 0;
+		}
+		for (int i=0; i<m; i++){
+			if ( X.get(i) > max ) {
+				max = X.get(i);
+			}
+		}
+		return max;
+	}
+
+	public static int sum(int[] X) {
+		int s = 0;
+		int m = X.length;
+		if ( m==0 ) {
+			return 0;
+		}
+		for (int i=0; i<m; i++){
+			s += X[i];
+		}
+		return s;
+	}
+
+	public static void writeReport(String filename, String data) throws IOException {
+		String absoluteFilename = getAbsoluteFileName(filename);
+		File f = new File(absoluteFilename);
+		File parent = f.getParentFile();
+		parent.mkdirs(); // ensure parent dir exists
+		// write file
+		FileOutputStream fos = new FileOutputStream(f, true); // append
+		BufferedOutputStream bos = new BufferedOutputStream(fos);
+		PrintStream ps = new PrintStream(bos);
+		ps.println(data);
+		ps.close();
+		bos.close();
+		fos.close();
+	}
+
+	public static String getAbsoluteFileName(String filename) {
+		return System.getProperty("user.home")+File.separator+
+					ANTLRWORKS_DIR +File.separator+
+					filename;
+	}
+}
diff --git a/src/org/antlr/runtime/tree/.DS_Store b/src/org/antlr/runtime/tree/.DS_Store
new file mode 100644
index 0000000..5008ddf
--- /dev/null
+++ b/src/org/antlr/runtime/tree/.DS_Store
diff --git a/src/org/antlr/runtime/tree/BaseTree.java b/src/org/antlr/runtime/tree/BaseTree.java
new file mode 100755
index 0000000..ebffdb3
--- /dev/null
+++ b/src/org/antlr/runtime/tree/BaseTree.java
@@ -0,0 +1,349 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** A generic tree implementation with no payload.  You must subclass to
+ *  actually have any user data.  ANTLR v3 uses a list of children approach
+ *  instead of the child-sibling approach in v2.  A flat tree (a list) is
+ *  an empty node whose children represent the list.  An empty, but
+ *  non-null node is called "nil".
+ */
+public abstract class BaseTree implements Tree {
+	protected List children;
+
+	public BaseTree() {
+	}
+
+	/** Create a new node from an existing node does nothing for BaseTree
+	 *  as there are no fields other than the children list, which cannot
+	 *  be copied as the children are not considered part of this node. 
+	 */
+	public BaseTree(Tree node) {
+	}
+
+	public Tree getChild(int i) {
+		if ( children==null || i>=children.size() ) {
+			return null;
+		}
+		return (Tree)children.get(i);
+	}
+
+	/** Get the children internal List; note that if you directly mess with
+	 *  the list, do so at your own risk.
+	 */
+	public List getChildren() {
+		return children;
+	}
+
+	public Tree getFirstChildWithType(int type) {
+		for (int i = 0; children!=null && i < children.size(); i++) {
+			Tree t = (Tree) children.get(i);
+			if ( t.getType()==type ) {
+				return t;
+			}
+		}	
+		return null;
+	}
+
+	public int getChildCount() {
+		if ( children==null ) {
+			return 0;
+		}
+		return children.size();
+	}
+
+	/** Add t as child of this node.
+	 *
+	 *  Warning: if t has no children, but child does
+	 *  and child isNil then this routine moves children to t via
+	 *  t.children = child.children; i.e., without copying the array.
+	 */
+	public void addChild(Tree t) {
+		//System.out.println("add child "+t.toStringTree()+" "+this.toStringTree());
+		//System.out.println("existing children: "+children);
+		if ( t==null ) {
+			return; // do nothing upon addChild(null)
+		}
+		BaseTree childTree = (BaseTree)t;
+		if ( childTree.isNil() ) { // t is an empty node possibly with children
+			if ( this.children!=null && this.children == childTree.children ) {
+				throw new RuntimeException("attempt to add child list to itself");
+			}
+			// just add all of childTree's children to this
+			if ( childTree.children!=null ) {
+				if ( this.children!=null ) { // must copy, this has children already
+					int n = childTree.children.size();
+					for (int i = 0; i < n; i++) {
+						Tree c = (Tree)childTree.children.get(i);
+						this.children.add(c);
+						// handle double-link stuff for each child of nil root
+						c.setParent(this);
+						c.setChildIndex(children.size()-1);
+					}
+				}
+				else {
+					// no children for this but t has children; just set pointer
+					// call general freshener routine
+					this.children = childTree.children;
+					this.freshenParentAndChildIndexes();
+				}
+			}
+		}
+		else { // child is not nil (don't care about children)
+			if ( children==null ) {
+				children = createChildrenList(); // create children list on demand
+			}
+			children.add(t);
+			childTree.setParent(this);
+			childTree.setChildIndex(children.size()-1);
+		}
+		// System.out.println("now children are: "+children);
+	}
+
+	/** Add all elements of kids list as children of this node */
+	public void addChildren(List kids) {
+		for (int i = 0; i < kids.size(); i++) {
+			Tree t = (Tree) kids.get(i);
+			addChild(t);
+		}
+	}
+
+	public void setChild(int i, Tree t) {
+		if ( t==null ) {
+			return;
+		}
+		if ( t.isNil() ) {
+			throw new IllegalArgumentException("Can't set single child to a list");
+		}
+		if ( children==null ) {
+			children = createChildrenList();
+		}
+		children.set(i, t);
+		t.setParent(this);
+		t.setChildIndex(i);
+	}
+	
+	public Object deleteChild(int i) {
+		if ( children==null ) {
+			return null;
+		}
+		Tree killed = (Tree)children.remove(i);
+		// walk rest and decrement their child indexes
+		this.freshenParentAndChildIndexes(i);
+		return killed;
+	}
+
+	/** Delete children from start to stop and replace with t even if t is
+	 *  a list (nil-root tree).  num of children can increase or decrease.
+	 *  For huge child lists, inserting children can force walking rest of
+	 *  children to set their childindex; could be slow.
+	 */
+	public void replaceChildren(int startChildIndex, int stopChildIndex, Object t) {
+		/*
+		System.out.println("replaceChildren "+startChildIndex+", "+stopChildIndex+
+						   " with "+((BaseTree)t).toStringTree());
+		System.out.println("in="+toStringTree());
+		*/
+		if ( children==null ) {
+			throw new IllegalArgumentException("indexes invalid; no children in list");
+		}
+		int replacingHowMany = stopChildIndex - startChildIndex + 1;
+		int replacingWithHowMany;
+		BaseTree newTree = (BaseTree)t;
+		List newChildren = null;
+		// normalize to a list of children to add: newChildren
+		if ( newTree.isNil() ) {
+			newChildren = newTree.children;
+		}
+		else {
+			newChildren = new ArrayList(1);
+			newChildren.add(newTree);
+		}
+		replacingWithHowMany = newChildren.size();
+		int numNewChildren = newChildren.size();
+		int delta = replacingHowMany - replacingWithHowMany;
+		// if same number of nodes, do direct replace
+		if ( delta == 0 ) {
+			int j = 0; // index into new children
+			for (int i=startChildIndex; i<=stopChildIndex; i++) {
+				BaseTree child = (BaseTree)newChildren.get(j);
+				children.set(i, child);
+				child.setParent(this);
+				child.setChildIndex(i);
+                j++;
+            }
+		}
+		else if ( delta > 0 ) { // fewer new nodes than there were
+			// set children and then delete extra
+			for (int j=0; j<numNewChildren; j++) {
+				children.set(startChildIndex+j, newChildren.get(j));
+			}
+			int indexToDelete = startChildIndex+numNewChildren;
+			for (int c=indexToDelete; c<=stopChildIndex; c++) {
+				// delete same index, shifting everybody down each time
+				children.remove(indexToDelete);
+			}
+			freshenParentAndChildIndexes(startChildIndex);
+		}
+		else { // more new nodes than were there before
+			// fill in as many children as we can (replacingHowMany) w/o moving data
+			for (int j=0; j<replacingHowMany; j++) {
+				children.set(startChildIndex+j, newChildren.get(j));
+			}
+			int numToInsert = replacingWithHowMany-replacingHowMany;
+			for (int j=replacingHowMany; j<replacingWithHowMany; j++) {
+				children.add(startChildIndex+j, newChildren.get(j));
+			}
+			freshenParentAndChildIndexes(startChildIndex);
+		}
+		//System.out.println("out="+toStringTree());
+	}
+
+	/** Override in a subclass to change the impl of children list */
+	protected List createChildrenList() {
+		return new ArrayList();
+	}
+
+	public boolean isNil() {
+		return false;
+	}
+
+	/** Set the parent and child index values for all child of t */
+	public void freshenParentAndChildIndexes() {
+		freshenParentAndChildIndexes(0);
+	}
+
+	public void freshenParentAndChildIndexes(int offset) {
+		int n = getChildCount();
+		for (int c = offset; c < n; c++) {
+			Tree child = (Tree)getChild(c);
+			child.setChildIndex(c);
+			child.setParent(this);
+		}
+	}
+
+	public void sanityCheckParentAndChildIndexes() {
+		sanityCheckParentAndChildIndexes(null, -1);
+	}
+
+	public void sanityCheckParentAndChildIndexes(Tree parent, int i) {
+		if ( parent!=this.getParent() ) {
+			throw new IllegalStateException("parents don't match; expected "+parent+" found "+this.getParent());
+		}
+		if ( i!=this.getChildIndex() ) {
+			throw new IllegalStateException("child indexes don't match; expected "+i+" found "+this.getChildIndex());
+		}
+		int n = this.getChildCount();
+		for (int c = 0; c < n; c++) {
+			CommonTree child = (CommonTree)this.getChild(c);
+			child.sanityCheckParentAndChildIndexes(this, c);
+		}
+	}
+
+	/** BaseTree doesn't track child indexes. */
+	public int getChildIndex() {
+		return 0;
+	}
+	public void setChildIndex(int index) {
+	}
+
+	/** BaseTree doesn't track parent pointers. */
+	public Tree getParent() {
+		return null;
+	}
+
+    public void setParent(Tree t) {
+	}
+
+    /** Walk upwards looking for ancestor with this token type. */
+    public boolean hasAncestor(int ttype) { return getAncestor(ttype)!=null; }
+
+    /** Walk upwards and get first ancestor with this token type. */
+    public Tree getAncestor(int ttype) {
+        Tree t = this;
+        t = t.getParent();
+        while ( t!=null ) {
+            if ( t.getType()==ttype ) return t;
+            t = t.getParent();
+        }
+        return null;
+    }
+
+    /** Return a list of all ancestors of this node.  The first node of
+     *  list is the root and the last is the parent of this node.
+     */
+    public List getAncestors() {
+        if ( getParent()==null ) return null;
+        List ancestors = new ArrayList();
+        Tree t = this;
+        t = t.getParent();
+        while ( t!=null ) {
+            ancestors.add(0, t); // insert at start
+            t = t.getParent();
+        }
+        return ancestors;
+    }
+
+    /** Print out a whole tree not just a node */
+    public String toStringTree() {
+		if ( children==null || children.size()==0 ) {
+			return this.toString();
+		}
+		StringBuffer buf = new StringBuffer();
+		if ( !isNil() ) {
+			buf.append("(");
+			buf.append(this.toString());
+			buf.append(' ');
+		}
+		for (int i = 0; children!=null && i < children.size(); i++) {
+			Tree t = (Tree)children.get(i);
+			if ( i>0 ) {
+				buf.append(' ');
+			}
+			buf.append(t.toStringTree());
+		}
+		if ( !isNil() ) {
+			buf.append(")");
+		}
+		return buf.toString();
+	}
+
+    public int getLine() {
+		return 0;
+	}
+
+	public int getCharPositionInLine() {
+		return 0;
+	}
+
+	/** Override to say how a node (not a tree) should look as text */
+	public abstract String toString();
+}
diff --git a/src/org/antlr/runtime/tree/BaseTreeAdaptor.java b/src/org/antlr/runtime/tree/BaseTreeAdaptor.java
new file mode 100755
index 0000000..33140b1
--- /dev/null
+++ b/src/org/antlr/runtime/tree/BaseTreeAdaptor.java
@@ -0,0 +1,279 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.RecognitionException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** A TreeAdaptor that works with any Tree implementation. */
+public abstract class BaseTreeAdaptor implements TreeAdaptor {
+	/** System.identityHashCode() is not always unique; we have to
+	 *  track ourselves.  That's ok, it's only for debugging, though it's
+	 *  expensive: we have to create a hashtable with all tree nodes in it.
+	 */
+	protected Map treeToUniqueIDMap;
+	protected int uniqueNodeID = 1;
+
+	public Object nil() {
+		return create(null);
+	}
+
+	/** create tree node that holds the start and stop tokens associated
+	 *  with an error.
+	 *
+	 *  If you specify your own kind of tree nodes, you will likely have to
+	 *  override this method. CommonTree returns Token.INVALID_TOKEN_TYPE
+	 *  if no token payload but you might have to set token type for diff
+	 *  node type.
+     *
+     *  You don't have to subclass CommonErrorNode; you will likely need to
+     *  subclass your own tree node class to avoid class cast exception.
+	 */
+	public Object errorNode(TokenStream input, Token start, Token stop,
+							RecognitionException e)
+	{
+		CommonErrorNode t = new CommonErrorNode(input, start, stop, e);
+		//System.out.println("returning error node '"+t+"' @index="+input.index());
+		return t;
+	}
+
+	public boolean isNil(Object tree) {
+		return ((Tree)tree).isNil();
+	}
+
+	public Object dupTree(Object tree) {
+		return dupTree(tree, null);
+	}
+
+	/** This is generic in the sense that it will work with any kind of
+	 *  tree (not just Tree interface).  It invokes the adaptor routines
+	 *  not the tree node routines to do the construction.  
+	 */
+	public Object dupTree(Object t, Object parent) {
+		if ( t==null ) {
+			return null;
+		}
+		Object newTree = dupNode(t);
+		// ensure new subtree root has parent/child index set
+		setChildIndex(newTree, getChildIndex(t)); // same index in new tree
+		setParent(newTree, parent);
+		int n = getChildCount(t);
+		for (int i = 0; i < n; i++) {
+			Object child = getChild(t, i);
+			Object newSubTree = dupTree(child, t);
+			addChild(newTree, newSubTree);
+		}
+		return newTree;
+	}
+
+	/** Add a child to the tree t.  If child is a flat tree (a list), make all
+	 *  in list children of t.  Warning: if t has no children, but child does
+	 *  and child isNil then you can decide it is ok to move children to t via
+	 *  t.children = child.children; i.e., without copying the array.  Just
+	 *  make sure that this is consistent with have the user will build
+	 *  ASTs.
+	 */
+	public void addChild(Object t, Object child) {
+		if ( t!=null && child!=null ) {
+			((Tree)t).addChild((Tree)child);
+		}
+	}
+
+	/** If oldRoot is a nil root, just copy or move the children to newRoot.
+	 *  If not a nil root, make oldRoot a child of newRoot.
+	 *
+	 *    old=^(nil a b c), new=r yields ^(r a b c)
+	 *    old=^(a b c), new=r yields ^(r ^(a b c))
+	 *
+	 *  If newRoot is a nil-rooted single child tree, use the single
+	 *  child as the new root node.
+	 *
+	 *    old=^(nil a b c), new=^(nil r) yields ^(r a b c)
+	 *    old=^(a b c), new=^(nil r) yields ^(r ^(a b c))
+	 *
+	 *  If oldRoot was null, it's ok, just return newRoot (even if isNil).
+	 *
+	 *    old=null, new=r yields r
+	 *    old=null, new=^(nil r) yields ^(nil r)
+	 *
+	 *  Return newRoot.  Throw an exception if newRoot is not a
+	 *  simple node or nil root with a single child node--it must be a root
+	 *  node.  If newRoot is ^(nil x) return x as newRoot.
+	 *
+	 *  Be advised that it's ok for newRoot to point at oldRoot's
+	 *  children; i.e., you don't have to copy the list.  We are
+	 *  constructing these nodes so we should have this control for
+	 *  efficiency.
+	 */
+	public Object becomeRoot(Object newRoot, Object oldRoot) {
+        //System.out.println("becomeroot new "+newRoot.toString()+" old "+oldRoot);
+        Tree newRootTree = (Tree)newRoot;
+		Tree oldRootTree = (Tree)oldRoot;
+		if ( oldRoot==null ) {
+			return newRoot;
+		}
+		// handle ^(nil real-node)
+		if ( newRootTree.isNil() ) {
+            int nc = newRootTree.getChildCount();
+            if ( nc==1 ) newRootTree = (Tree)newRootTree.getChild(0);
+            else if ( nc >1 ) {
+				// TODO: make tree run time exceptions hierarchy
+				throw new RuntimeException("more than one node as root (TODO: make exception hierarchy)");
+			}
+        }
+		// add oldRoot to newRoot; addChild takes care of case where oldRoot
+		// is a flat list (i.e., nil-rooted tree).  All children of oldRoot
+		// are added to newRoot.
+		newRootTree.addChild(oldRootTree);
+		return newRootTree;
+	}
+
+	/** Transform ^(nil x) to x and nil to null */
+	public Object rulePostProcessing(Object root) {
+		//System.out.println("rulePostProcessing: "+((Tree)root).toStringTree());
+		Tree r = (Tree)root;
+		if ( r!=null && r.isNil() ) {
+			if ( r.getChildCount()==0 ) {
+				r = null;
+			}
+			else if ( r.getChildCount()==1 ) {
+				r = (Tree)r.getChild(0);
+				// whoever invokes rule will set parent and child index
+				r.setParent(null);
+				r.setChildIndex(-1);
+			}
+		}
+		return r;
+	}
+
+	public Object becomeRoot(Token newRoot, Object oldRoot) {
+		return becomeRoot(create(newRoot), oldRoot);
+	}
+
+	public Object create(int tokenType, Token fromToken) {
+		fromToken = createToken(fromToken);
+		//((ClassicToken)fromToken).setType(tokenType);
+		fromToken.setType(tokenType);
+		Tree t = (Tree)create(fromToken);
+		return t;
+	}
+
+	public Object create(int tokenType, Token fromToken, String text) {
+        if (fromToken == null) return create(tokenType, text);
+		fromToken = createToken(fromToken);
+		fromToken.setType(tokenType);
+		fromToken.setText(text);
+		Tree t = (Tree)create(fromToken);
+		return t;
+	}
+
+	public Object create(int tokenType, String text) {
+		Token fromToken = createToken(tokenType, text);
+		Tree t = (Tree)create(fromToken);
+		return t;
+	}
+
+	public int getType(Object t) {
+		return ((Tree)t).getType();
+	}
+
+	public void setType(Object t, int type) {
+		throw new NoSuchMethodError("don't know enough about Tree node");
+	}
+
+	public String getText(Object t) {
+		return ((Tree)t).getText();
+	}
+
+	public void setText(Object t, String text) {
+		throw new NoSuchMethodError("don't know enough about Tree node");
+	}
+
+	public Object getChild(Object t, int i) {
+		return ((Tree)t).getChild(i);
+	}
+
+	public void setChild(Object t, int i, Object child) {
+		((Tree)t).setChild(i, (Tree)child);
+	}
+
+	public Object deleteChild(Object t, int i) {
+		return ((Tree)t).deleteChild(i);
+	}
+
+	public int getChildCount(Object t) {
+		return ((Tree)t).getChildCount();
+	}
+
+	public int getUniqueID(Object node) {
+		if ( treeToUniqueIDMap==null ) {
+			 treeToUniqueIDMap = new HashMap();
+		}
+		Integer prevID = (Integer)treeToUniqueIDMap.get(node);
+		if ( prevID!=null ) {
+			return prevID.intValue();
+		}
+		int ID = uniqueNodeID;
+		treeToUniqueIDMap.put(node, new Integer(ID));
+		uniqueNodeID++;
+		return ID;
+		// GC makes these nonunique:
+		// return System.identityHashCode(node);
+	}
+
+	/** Tell me how to create a token for use with imaginary token nodes.
+	 *  For example, there is probably no input symbol associated with imaginary
+	 *  token DECL, but you need to create it as a payload or whatever for
+	 *  the DECL node as in ^(DECL type ID).
+	 *
+	 *  If you care what the token payload objects' type is, you should
+	 *  override this method and any other createToken variant.
+	 */
+	public abstract Token createToken(int tokenType, String text);
+
+	/** Tell me how to create a token for use with imaginary token nodes.
+	 *  For example, there is probably no input symbol associated with imaginary
+	 *  token DECL, but you need to create it as a payload or whatever for
+	 *  the DECL node as in ^(DECL type ID).
+	 *
+	 *  This is a variant of createToken where the new token is derived from
+	 *  an actual real input token.  Typically this is for converting '{'
+	 *  tokens to BLOCK etc...  You'll see
+	 *
+	 *    r : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID+) ;
+	 *
+	 *  If you care what the token payload objects' type is, you should
+	 *  override this method and any other createToken variant.
+	 */
+	public abstract Token createToken(Token fromToken);
+}
+
diff --git a/src/org/antlr/runtime/tree/BufferedTreeNodeStream.java b/src/org/antlr/runtime/tree/BufferedTreeNodeStream.java
new file mode 100755
index 0000000..d9a2a7e
--- /dev/null
+++ b/src/org/antlr/runtime/tree/BufferedTreeNodeStream.java
@@ -0,0 +1,478 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.misc.IntArray;
+import java.util.*;
+
+/** A buffered stream of tree nodes.  Nodes can be from a tree of ANY kind.
+ *
+ *  This node stream sucks all nodes out of the tree specified in
+ *  the constructor during construction and makes pointers into
+ *  the tree using an array of Object pointers. The stream necessarily
+ *  includes pointers to DOWN and UP and EOF nodes.
+ *
+ *  This stream knows how to mark/release for backtracking.
+ *
+ *  This stream is most suitable for tree interpreters that need to
+ *  jump around a lot or for tree parsers requiring speed (at cost of memory).
+ *  There is some duplicated functionality here with UnBufferedTreeNodeStream
+ *  but just in bookkeeping, not tree walking etc...
+ *
+ *  TARGET DEVELOPERS:
+ *
+ *  This is the old CommonTreeNodeStream that buffered up entire node stream.
+ *  No need to implement really as new CommonTreeNodeStream is much better
+ *  and covers what we need.
+ *
+ *  @see CommonTreeNodeStream
+ */
+public class BufferedTreeNodeStream implements TreeNodeStream {
+	public static final int DEFAULT_INITIAL_BUFFER_SIZE = 100;
+	public static final int INITIAL_CALL_STACK_SIZE = 10;
+
+    protected class StreamIterator implements Iterator {
+		int i = 0;
+		public boolean hasNext() {
+			return i<nodes.size();
+		}
+
+		public Object next() {
+			int current = i;
+			i++;
+			if ( current < nodes.size() ) {
+				return nodes.get(current);
+			}
+			return eof;
+		}
+
+		public void remove() {
+			throw new RuntimeException("cannot remove nodes from stream");
+		}
+	}
+
+	// all these navigation nodes are shared and hence they
+	// cannot contain any line/column info
+
+	protected Object down;
+	protected Object up;
+	protected Object eof;
+
+	/** The complete mapping from stream index to tree node.
+	 *  This buffer includes pointers to DOWN, UP, and EOF nodes.
+	 *  It is built upon ctor invocation.  The elements are type
+	 *  Object as we don't what the trees look like.
+	 *
+	 *  Load upon first need of the buffer so we can set token types
+	 *  of interest for reverseIndexing.  Slows us down a wee bit to
+	 *  do all of the if p==-1 testing everywhere though.
+	 */
+	protected List nodes;
+
+	/** Pull nodes from which tree? */
+	protected Object root;
+
+	/** IF this tree (root) was created from a token stream, track it. */
+	protected TokenStream tokens;
+
+	/** What tree adaptor was used to build these trees */
+	TreeAdaptor adaptor;
+
+	/** Reuse same DOWN, UP navigation nodes unless this is true */
+	protected boolean uniqueNavigationNodes = false;
+
+	/** The index into the nodes list of the current node (next node
+	 *  to consume).  If -1, nodes array not filled yet.
+	 */
+	protected int p = -1;
+
+	/** Track the last mark() call result value for use in rewind(). */
+	protected int lastMarker;
+
+	/** Stack of indexes used for push/pop calls */
+	protected IntArray calls;
+
+	public BufferedTreeNodeStream(Object tree) {
+		this(new CommonTreeAdaptor(), tree);
+	}
+
+	public BufferedTreeNodeStream(TreeAdaptor adaptor, Object tree) {
+		this(adaptor, tree, DEFAULT_INITIAL_BUFFER_SIZE);
+	}
+
+	public BufferedTreeNodeStream(TreeAdaptor adaptor, Object tree, int initialBufferSize) {
+		this.root = tree;
+		this.adaptor = adaptor;
+		nodes = new ArrayList(initialBufferSize);
+		down = adaptor.create(Token.DOWN, "DOWN");
+		up = adaptor.create(Token.UP, "UP");
+		eof = adaptor.create(Token.EOF, "EOF");
+	}
+
+	/** Walk tree with depth-first-search and fill nodes buffer.
+	 *  Don't do DOWN, UP nodes if its a list (t is isNil).
+	 */
+	protected void fillBuffer() {
+		fillBuffer(root);
+		//System.out.println("revIndex="+tokenTypeToStreamIndexesMap);
+		p = 0; // buffer of nodes intialized now
+	}
+
+	public void fillBuffer(Object t) {
+		boolean nil = adaptor.isNil(t);
+		if ( !nil ) {
+			nodes.add(t); // add this node
+		}
+		// add DOWN node if t has children
+		int n = adaptor.getChildCount(t);
+		if ( !nil && n>0 ) {
+			addNavigationNode(Token.DOWN);
+		}
+		// and now add all its children
+		for (int c=0; c<n; c++) {
+			Object child = adaptor.getChild(t,c);
+			fillBuffer(child);
+		}
+		// add UP node if t has children
+		if ( !nil && n>0 ) {
+			addNavigationNode(Token.UP);
+		}
+	}
+
+	/** What is the stream index for node? 0..n-1
+	 *  Return -1 if node not found.
+	 */
+	protected int getNodeIndex(Object node) {
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		for (int i = 0; i < nodes.size(); i++) {
+			Object t = (Object) nodes.get(i);
+			if ( t==node ) {
+				return i;
+			}
+		}
+		return -1;
+	}
+
+	/** As we flatten the tree, we use UP, DOWN nodes to represent
+	 *  the tree structure.  When debugging we need unique nodes
+	 *  so instantiate new ones when uniqueNavigationNodes is true.
+	 */
+	protected void addNavigationNode(final int ttype) {
+		Object navNode = null;
+		if ( ttype==Token.DOWN ) {
+			if ( hasUniqueNavigationNodes() ) {
+				navNode = adaptor.create(Token.DOWN, "DOWN");
+			}
+			else {
+				navNode = down;
+			}
+		}
+		else {
+			if ( hasUniqueNavigationNodes() ) {
+				navNode = adaptor.create(Token.UP, "UP");
+			}
+			else {
+				navNode = up;
+			}
+		}
+		nodes.add(navNode);
+	}
+
+	public Object get(int i) {
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		return nodes.get(i);
+	}
+
+	public Object LT(int k) {
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		if ( k==0 ) {
+			return null;
+		}
+		if ( k<0 ) {
+			return LB(-k);
+		}
+		//System.out.print("LT(p="+p+","+k+")=");
+		if ( (p+k-1) >= nodes.size() ) {
+			return eof;
+		}
+		return nodes.get(p+k-1);
+	}
+
+	public Object getCurrentSymbol() { return LT(1); }
+
+/*
+	public Object getLastTreeNode() {
+		int i = index();
+		if ( i>=size() ) {
+			i--; // if at EOF, have to start one back
+		}
+		System.out.println("start last node: "+i+" size=="+nodes.size());
+		while ( i>=0 &&
+			(adaptor.getType(get(i))==Token.EOF ||
+			 adaptor.getType(get(i))==Token.UP ||
+			 adaptor.getType(get(i))==Token.DOWN) )
+		{
+			i--;
+		}
+		System.out.println("stop at node: "+i+" "+nodes.get(i));
+		return nodes.get(i);
+	}
+*/
+	
+	/** Look backwards k nodes */
+	protected Object LB(int k) {
+		if ( k==0 ) {
+			return null;
+		}
+		if ( (p-k)<0 ) {
+			return null;
+		}
+		return nodes.get(p-k);
+	}
+
+	public Object getTreeSource() {
+		return root;
+	}
+
+	public String getSourceName() {
+		return getTokenStream().getSourceName();
+	}
+
+	public TokenStream getTokenStream() {
+		return tokens;
+	}
+
+	public void setTokenStream(TokenStream tokens) {
+		this.tokens = tokens;
+	}
+
+	public TreeAdaptor getTreeAdaptor() {
+		return adaptor;
+	}
+
+	public void setTreeAdaptor(TreeAdaptor adaptor) {
+		this.adaptor = adaptor;
+	}
+
+	public boolean hasUniqueNavigationNodes() {
+		return uniqueNavigationNodes;
+	}
+
+	public void setUniqueNavigationNodes(boolean uniqueNavigationNodes) {
+		this.uniqueNavigationNodes = uniqueNavigationNodes;
+	}
+
+	public void consume() {
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		p++;
+	}
+
+	public int LA(int i) {
+		return adaptor.getType(LT(i));
+	}
+
+	public int mark() {
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		lastMarker = index();
+		return lastMarker;
+	}
+
+	public void release(int marker) {
+		// no resources to release
+	}
+
+	public int index() {
+		return p;
+	}
+
+	public void rewind(int marker) {
+		seek(marker);
+	}
+
+	public void rewind() {
+		seek(lastMarker);
+	}
+
+	public void seek(int index) {
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		p = index;
+	}
+
+	/** Make stream jump to a new location, saving old location.
+	 *  Switch back with pop().
+	 */
+	public void push(int index) {
+		if ( calls==null ) {
+			calls = new IntArray();
+		}
+		calls.push(p); // save current index
+		seek(index);
+	}
+
+	/** Seek back to previous index saved during last push() call.
+	 *  Return top of stack (return index).
+	 */
+	public int pop() {
+		int ret = calls.pop();
+		seek(ret);
+		return ret;
+	}
+
+	public void reset() {
+		p = 0;
+		lastMarker = 0;
+        if (calls != null) {
+            calls.clear();
+        }
+    }
+
+	public int size() {
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		return nodes.size();
+	}
+
+	public Iterator iterator() {
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		return new StreamIterator();
+	}
+
+	// TREE REWRITE INTERFACE
+
+	public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t) {
+		if ( parent!=null ) {
+			adaptor.replaceChildren(parent, startChildIndex, stopChildIndex, t);
+		}
+	}
+
+	/** Used for testing, just return the token type stream */
+	public String toTokenTypeString() {
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		StringBuffer buf = new StringBuffer();
+		for (int i = 0; i < nodes.size(); i++) {
+			Object t = (Object) nodes.get(i);
+			buf.append(" ");
+			buf.append(adaptor.getType(t));
+		}
+		return buf.toString();
+	}
+
+	/** Debugging */
+	public String toTokenString(int start, int stop) {
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		StringBuffer buf = new StringBuffer();
+		for (int i = start; i < nodes.size() && i <= stop; i++) {
+			Object t = (Object) nodes.get(i);
+			buf.append(" ");
+			buf.append(adaptor.getToken(t));
+		}
+		return buf.toString();
+	}
+
+	public String toString(Object start, Object stop) {
+		System.out.println("toString");
+		if ( start==null || stop==null ) {
+			return null;
+		}
+		if ( p==-1 ) {
+			fillBuffer();
+		}
+		//System.out.println("stop: "+stop);
+		if ( start instanceof CommonTree )
+			System.out.print("toString: "+((CommonTree)start).getToken()+", ");
+		else
+			System.out.println(start);
+		if ( stop instanceof CommonTree )
+			System.out.println(((CommonTree)stop).getToken());
+		else
+			System.out.println(stop);
+		// if we have the token stream, use that to dump text in order
+		if ( tokens!=null ) {
+			int beginTokenIndex = adaptor.getTokenStartIndex(start);
+			int endTokenIndex = adaptor.getTokenStopIndex(stop);
+			// if it's a tree, use start/stop index from start node
+			// else use token range from start/stop nodes
+			if ( adaptor.getType(stop)==Token.UP ) {
+				endTokenIndex = adaptor.getTokenStopIndex(start);
+			}
+			else if ( adaptor.getType(stop)==Token.EOF ) {
+				endTokenIndex = size()-2; // don't use EOF
+			}
+			return tokens.toString(beginTokenIndex, endTokenIndex);
+		}
+		// walk nodes looking for start
+		Object t = null;
+		int i = 0;
+		for (; i < nodes.size(); i++) {
+			t = nodes.get(i);
+			if ( t==start ) {
+				break;
+			}
+		}
+		// now walk until we see stop, filling string buffer with text
+		 StringBuffer buf = new StringBuffer();
+		t = nodes.get(i);
+		while ( t!=stop ) {
+			String text = adaptor.getText(t);
+			if ( text==null ) {
+				text = " "+String.valueOf(adaptor.getType(t));
+			}
+			buf.append(text);
+			i++;
+			t = nodes.get(i);
+		}
+		// include stop node too
+		String text = adaptor.getText(stop);
+		if ( text==null ) {
+			text = " "+String.valueOf(adaptor.getType(stop));
+		}
+		buf.append(text);
+		return buf.toString();
+	}
+}
diff --git a/src/org/antlr/runtime/tree/CommonErrorNode.java b/src/org/antlr/runtime/tree/CommonErrorNode.java
new file mode 100755
index 0000000..26b9933
--- /dev/null
+++ b/src/org/antlr/runtime/tree/CommonErrorNode.java
@@ -0,0 +1,108 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.*;
+
+/** A node representing erroneous token range in token stream */
+public class CommonErrorNode extends CommonTree {
+	public IntStream input;
+	public Token start;
+	public Token stop;
+	public RecognitionException trappedException;
+
+	public CommonErrorNode(TokenStream input, Token start, Token stop,
+						   RecognitionException e)
+	{
+		//System.out.println("start: "+start+", stop: "+stop);
+		if ( stop==null ||
+			 (stop.getTokenIndex() < start.getTokenIndex() &&
+			  stop.getType()!=Token.EOF) )
+		{
+			// sometimes resync does not consume a token (when LT(1) is
+			// in follow set.  So, stop will be 1 to left to start. adjust.
+			// Also handle case where start is the first token and no token
+			// is consumed during recovery; LT(-1) will return null.
+			stop = start;
+		}
+		this.input = input;
+		this.start = start;
+		this.stop = stop;
+		this.trappedException = e;
+	}
+
+	public boolean isNil() {
+		return false;
+	}
+
+	public int getType() {
+		return Token.INVALID_TOKEN_TYPE;
+	}
+
+	public String getText() {
+		String badText = null;
+		if ( start instanceof Token ) {
+			int i = ((Token)start).getTokenIndex();
+			int j = ((Token)stop).getTokenIndex();
+			if ( ((Token)stop).getType() == Token.EOF ) {
+				j = ((TokenStream)input).size();
+			}
+			badText = ((TokenStream)input).toString(i, j);
+		}
+		else if ( start instanceof Tree ) {
+			badText = ((TreeNodeStream)input).toString(start, stop);
+		}
+		else {
+			// people should subclass if they alter the tree type so this
+			// next one is for sure correct.
+			badText = "<unknown>";
+		}
+		return badText;
+	}
+
+	public String toString() {
+		if ( trappedException instanceof MissingTokenException ) {
+			return "<missing type: "+
+				   ((MissingTokenException)trappedException).getMissingType()+
+				   ">";
+		}
+		else if ( trappedException instanceof UnwantedTokenException ) {
+			return "<extraneous: "+
+				   ((UnwantedTokenException)trappedException).getUnexpectedToken()+
+				   ", resync="+getText()+">";
+		}
+		else if ( trappedException instanceof MismatchedTokenException ) {
+			return "<mismatched token: "+trappedException.token+", resync="+getText()+">";
+		}
+		else if ( trappedException instanceof NoViableAltException ) {
+			return "<unexpected: "+trappedException.token+
+				   ", resync="+getText()+">";
+		}
+		return "<error: "+getText()+">";
+	}
+}
diff --git a/src/org/antlr/runtime/tree/CommonTree.java b/src/org/antlr/runtime/tree/CommonTree.java
new file mode 100755
index 0000000..91c59de
--- /dev/null
+++ b/src/org/antlr/runtime/tree/CommonTree.java
@@ -0,0 +1,185 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+
+/** A tree node that is wrapper for a Token object.  After 3.0 release
+ *  while building tree rewrite stuff, it became clear that computing
+ *  parent and child index is very difficult and cumbersome.  Better to
+ *  spend the space in every tree node.  If you don't want these extra
+ *  fields, it's easy to cut them out in your own BaseTree subclass.
+ */
+public class CommonTree extends BaseTree {
+	/** A single token is the payload */
+	public Token token;
+
+	/** What token indexes bracket all tokens associated with this node
+	 *  and below?
+	 */
+	protected int startIndex=-1, stopIndex=-1;
+
+	/** Who is the parent node of this node; if null, implies node is root */
+	public CommonTree parent;
+
+	/** What index is this node in the child list? Range: 0..n-1 */
+	public int childIndex = -1;
+
+	public CommonTree() { }
+	
+	public CommonTree(CommonTree node) {
+		super(node);
+		this.token = node.token;
+		this.startIndex = node.startIndex;
+		this.stopIndex = node.stopIndex;
+	}
+
+	public CommonTree(Token t) {
+		this.token = t;
+	}
+
+	public Token getToken() {
+		return token;
+	}
+
+	public Tree dupNode() {
+		return new CommonTree(this);
+	}
+
+	public boolean isNil() {
+		return token==null;
+	}
+
+	public int getType() {
+		if ( token==null ) {
+			return Token.INVALID_TOKEN_TYPE;
+		}
+		return token.getType();
+	}
+
+	public String getText() {
+		if ( token==null ) {
+			return null;
+		}
+		return token.getText();
+	}
+
+	public int getLine() {
+		if ( token==null || token.getLine()==0 ) {
+			if ( getChildCount()>0 ) {
+				return getChild(0).getLine();
+			}
+			return 0;
+		}
+		return token.getLine();
+	}
+
+	public int getCharPositionInLine() {
+		if ( token==null || token.getCharPositionInLine()==-1 ) {
+			if ( getChildCount()>0 ) {
+				return getChild(0).getCharPositionInLine();
+			}
+			return 0;
+		}
+		return token.getCharPositionInLine();
+	}
+
+	public int getTokenStartIndex() {
+		if ( startIndex==-1 && token!=null ) {
+			return token.getTokenIndex();
+		}
+		return startIndex;
+	}
+
+	public void setTokenStartIndex(int index) {
+		startIndex = index;
+	}
+
+	public int getTokenStopIndex() {
+		if ( stopIndex==-1 && token!=null ) {
+			return token.getTokenIndex();
+		}
+		return stopIndex;
+	}
+
+	public void setTokenStopIndex(int index) {
+		stopIndex = index;
+	}
+
+    /** For every node in this subtree, make sure it's start/stop token's
+     *  are set.  Walk depth first, visit bottom up.  Only updates nodes
+     *  with at least one token index < 0.
+     */
+    public void setUnknownTokenBoundaries() {
+        if ( children==null ) {
+            if ( startIndex<0 || stopIndex<0 ) {
+                startIndex = stopIndex = token.getTokenIndex();
+            }
+            return;
+        }
+        for (int i=0; i<children.size(); i++) {
+            ((CommonTree)children.get(i)).setUnknownTokenBoundaries();
+        }
+        if ( startIndex>=0 && stopIndex>=0 ) return; // already set
+        if ( children.size() > 0 ) {
+            CommonTree firstChild = (CommonTree)children.get(0);
+            CommonTree lastChild = (CommonTree)children.get(children.size()-1);
+            startIndex = firstChild.getTokenStartIndex();
+            stopIndex = lastChild.getTokenStopIndex();
+        }
+    }
+
+	public int getChildIndex() {
+		return childIndex;
+	}
+
+	public Tree getParent() {
+		return parent;
+	}
+
+	public void setParent(Tree t) {
+		this.parent = (CommonTree)t;
+	}
+
+	public void setChildIndex(int index) {
+		this.childIndex = index;
+	}
+
+	public String toString() {
+		if ( isNil() ) {
+			return "nil";
+		}
+		if ( getType()==Token.INVALID_TOKEN_TYPE ) {
+			return "<errornode>";
+		}
+		if ( token==null ) {
+			return null;
+		}
+		return token.getText();
+	}
+}
diff --git a/src/org/antlr/runtime/tree/CommonTreeAdaptor.java b/src/org/antlr/runtime/tree/CommonTreeAdaptor.java
new file mode 100755
index 0000000..ebf560b
--- /dev/null
+++ b/src/org/antlr/runtime/tree/CommonTreeAdaptor.java
@@ -0,0 +1,168 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.CommonToken;
+import org.antlr.runtime.Token;
+
+/** A TreeAdaptor that works with any Tree implementation.  It provides
+ *  really just factory methods; all the work is done by BaseTreeAdaptor.
+ *  If you would like to have different tokens created than ClassicToken
+ *  objects, you need to override this and then set the parser tree adaptor to
+ *  use your subclass.
+ *
+ *  To get your parser to build nodes of a different type, override
+ *  create(Token), errorNode(), and to be safe, YourTreeClass.dupNode().
+ *  dupNode is called to duplicate nodes during rewrite operations.
+ */
+public class CommonTreeAdaptor extends BaseTreeAdaptor {
+	/** Duplicate a node.  This is part of the factory;
+	 *	override if you want another kind of node to be built.
+	 *
+	 *  I could use reflection to prevent having to override this
+	 *  but reflection is slow.
+	 */
+	public Object dupNode(Object t) {
+		if ( t==null ) return null;
+		return ((Tree)t).dupNode();
+	}
+
+	public Object create(Token payload) {
+		return new CommonTree(payload);
+	}
+
+	/** Tell me how to create a token for use with imaginary token nodes.
+	 *  For example, there is probably no input symbol associated with imaginary
+	 *  token DECL, but you need to create it as a payload or whatever for
+	 *  the DECL node as in ^(DECL type ID).
+	 *
+	 *  If you care what the token payload objects' type is, you should
+	 *  override this method and any other createToken variant.
+	 */
+	public Token createToken(int tokenType, String text) {
+		return new CommonToken(tokenType, text);
+	}
+
+	/** Tell me how to create a token for use with imaginary token nodes.
+	 *  For example, there is probably no input symbol associated with imaginary
+	 *  token DECL, but you need to create it as a payload or whatever for
+	 *  the DECL node as in ^(DECL type ID).
+	 *
+	 *  This is a variant of createToken where the new token is derived from
+	 *  an actual real input token.  Typically this is for converting '{'
+	 *  tokens to BLOCK etc...  You'll see
+	 *
+	 *    r : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID+) ;
+	 *
+	 *  If you care what the token payload objects' type is, you should
+	 *  override this method and any other createToken variant.
+	 */
+	public Token createToken(Token fromToken) {
+		return new CommonToken(fromToken);
+	}
+
+	/** Track start/stop token for subtree root created for a rule.
+	 *  Only works with Tree nodes.  For rules that match nothing,
+	 *  seems like this will yield start=i and stop=i-1 in a nil node.
+	 *  Might be useful info so I'll not force to be i..i.
+	 */
+	public void setTokenBoundaries(Object t, Token startToken, Token stopToken) {
+		if ( t==null ) return;
+		int start = 0;
+		int stop = 0;
+		if ( startToken!=null ) start = startToken.getTokenIndex();
+		if ( stopToken!=null ) stop = stopToken.getTokenIndex();
+		((Tree)t).setTokenStartIndex(start);
+		((Tree)t).setTokenStopIndex(stop);
+	}
+
+	public int getTokenStartIndex(Object t) {
+		if ( t==null ) return -1;
+		return ((Tree)t).getTokenStartIndex();
+	}
+
+	public int getTokenStopIndex(Object t) {
+		if ( t==null ) return -1;
+		return ((Tree)t).getTokenStopIndex();
+	}
+
+	public String getText(Object t) {
+		if ( t==null ) return null;
+		return ((Tree)t).getText();
+	}
+
+    public int getType(Object t) {
+		if ( t==null ) return Token.INVALID_TOKEN_TYPE;
+		return ((Tree)t).getType();
+	}
+
+	/** What is the Token associated with this node?  If
+	 *  you are not using CommonTree, then you must
+	 *  override this in your own adaptor.
+	 */
+	public Token getToken(Object t) {
+		if ( t instanceof CommonTree ) {
+			return ((CommonTree)t).getToken();
+		}
+		return null; // no idea what to do
+	}
+
+	public Object getChild(Object t, int i) {
+		if ( t==null ) return null;
+        return ((Tree)t).getChild(i);
+    }
+
+    public int getChildCount(Object t) {
+		if ( t==null ) return 0;
+        return ((Tree)t).getChildCount();
+    }
+
+	public Object getParent(Object t) {
+		if ( t==null ) return null;
+        return ((Tree)t).getParent();
+	}
+
+	public void setParent(Object t, Object parent) {
+        if ( t!=null ) ((Tree)t).setParent((Tree)parent);
+	}
+
+	public int getChildIndex(Object t) {
+        if ( t==null ) return 0;
+		return ((Tree)t).getChildIndex();
+	}
+
+	public void setChildIndex(Object t, int index) {
+        if ( t!=null ) ((Tree)t).setChildIndex(index);
+	}
+
+	public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t) {
+		if ( parent!=null ) {
+			((Tree)parent).replaceChildren(startChildIndex, stopChildIndex, t);
+		}
+	}
+}
diff --git a/src/org/antlr/runtime/tree/CommonTreeNodeStream.java b/src/org/antlr/runtime/tree/CommonTreeNodeStream.java
new file mode 100755
index 0000000..05dbbdd
--- /dev/null
+++ b/src/org/antlr/runtime/tree/CommonTreeNodeStream.java
@@ -0,0 +1,171 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.misc.LookaheadStream;
+import org.antlr.runtime.misc.IntArray;
+
+import java.util.*;
+
+public class CommonTreeNodeStream extends LookaheadStream<Object> implements TreeNodeStream {
+	public static final int DEFAULT_INITIAL_BUFFER_SIZE = 100;
+	public static final int INITIAL_CALL_STACK_SIZE = 10;
+
+	/** Pull nodes from which tree? */
+	protected Object root;
+
+	/** If this tree (root) was created from a token stream, track it. */
+	protected TokenStream tokens;
+
+	/** What tree adaptor was used to build these trees */
+	TreeAdaptor adaptor;
+
+    /** The tree iterator we using */
+    protected TreeIterator it;
+
+    /** Stack of indexes used for push/pop calls */
+    protected IntArray calls;    
+
+    /** Tree (nil A B C) trees like flat A B C streams */
+    protected boolean hasNilRoot = false;
+
+    /** Tracks tree depth.  Level=0 means we're at root node level. */
+    protected int level = 0;
+
+	public CommonTreeNodeStream(Object tree) {
+		this(new CommonTreeAdaptor(), tree);
+	}
+
+	public CommonTreeNodeStream(TreeAdaptor adaptor, Object tree) {
+		this.root = tree;
+		this.adaptor = adaptor;
+        it = new TreeIterator(adaptor,root);
+	}
+
+    public void reset() {
+        super.reset();
+        it.reset();
+        hasNilRoot = false;
+        level = 0;
+        if ( calls != null ) calls.clear();
+    }
+    
+    /** Pull elements from tree iterator.  Track tree level 0..max_level.
+     *  If nil rooted tree, don't give initial nil and DOWN nor final UP.
+     */
+    public Object nextElement() {
+        Object t = it.next();
+        //System.out.println("pulled "+adaptor.getType(t));
+        if ( t == it.up ) {
+            level--;
+            if ( level==0 && hasNilRoot ) return it.next(); // don't give last UP; get EOF
+        }
+        else if ( t == it.down ) level++;
+        if ( level==0 && adaptor.isNil(t) ) { // if nil root, scarf nil, DOWN
+            hasNilRoot = true;
+            t = it.next(); // t is now DOWN, so get first real node next
+            level++;
+            t = it.next();
+        }
+        return t;
+    }
+
+    public boolean isEOF(Object o) { return adaptor.getType(o) == Token.EOF; }
+
+    public void setUniqueNavigationNodes(boolean uniqueNavigationNodes) { }
+
+	public Object getTreeSource() {	return root; }
+
+	public String getSourceName() { return getTokenStream().getSourceName(); }
+
+	public TokenStream getTokenStream() { return tokens; }
+
+	public void setTokenStream(TokenStream tokens) { this.tokens = tokens; }
+
+	public TreeAdaptor getTreeAdaptor() { return adaptor; }
+
+	public void setTreeAdaptor(TreeAdaptor adaptor) { this.adaptor = adaptor; }
+
+    public Object get(int i) {
+        throw new UnsupportedOperationException("Absolute node indexes are meaningless in an unbuffered stream");
+    }
+
+    public int LA(int i) { return adaptor.getType(LT(i)); }
+
+    /** Make stream jump to a new location, saving old location.
+     *  Switch back with pop().
+     */
+    public void push(int index) {
+        if ( calls==null ) {
+            calls = new IntArray();
+        }
+        calls.push(p); // save current index
+        seek(index);
+    }
+
+    /** Seek back to previous index saved during last push() call.
+     *  Return top of stack (return index).
+     */
+    public int pop() {
+        int ret = calls.pop();
+        seek(ret);
+        return ret;
+    }    
+
+	// TREE REWRITE INTERFACE
+
+	public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t) {
+		if ( parent!=null ) {
+			adaptor.replaceChildren(parent, startChildIndex, stopChildIndex, t);
+		}
+	}
+
+	public String toString(Object start, Object stop) {
+        // we'll have to walk from start to stop in tree; we're not keeping
+        // a complete node stream buffer
+        return "n/a";
+	}
+
+    /** For debugging; destructive: moves tree iterator to end. */
+    public String toTokenTypeString() {
+        reset();
+		StringBuffer buf = new StringBuffer();
+        Object o = LT(1);
+        int type = adaptor.getType(o);
+        while ( type!=Token.EOF ) {
+            buf.append(" ");
+            buf.append(type);
+            consume();
+            o = LT(1);
+            type = adaptor.getType(o);
+		}
+		return buf.toString();
+    }
+}
diff --git a/src/org/antlr/runtime/tree/ParseTree.java b/src/org/antlr/runtime/tree/ParseTree.java
new file mode 100755
index 0000000..5811c55
--- /dev/null
+++ b/src/org/antlr/runtime/tree/ParseTree.java
@@ -0,0 +1,119 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+
+import java.util.List;
+
+/** A record of the rules used to match a token sequence.  The tokens
+ *  end up as the leaves of this tree and rule nodes are the interior nodes.
+ *  This really adds no functionality, it is just an alias for CommonTree
+ *  that is more meaningful (specific) and holds a String to display for a node.
+ */
+public class ParseTree extends BaseTree {
+	public Object payload;
+	public List hiddenTokens;
+
+	public ParseTree(Object label) {
+		this.payload = label;
+	}
+
+	public Tree dupNode() {
+		return null;
+	}
+
+	public int getType() {
+		return 0;
+	}
+
+	public String getText() {
+		return toString();
+	}
+
+	public int getTokenStartIndex() {
+		return 0;
+	}
+
+	public void setTokenStartIndex(int index) {
+	}
+
+	public int getTokenStopIndex() {
+		return 0;
+	}
+
+	public void setTokenStopIndex(int index) {
+	}
+
+	public String toString() {
+		if ( payload instanceof Token ) {
+			Token t = (Token)payload;
+			if ( t.getType() == Token.EOF ) {
+				return "<EOF>";
+			}
+			return t.getText();
+		}
+		return payload.toString();
+	}
+
+	/** Emit a token and all hidden nodes before.  EOF node holds all
+	 *  hidden tokens after last real token.
+	 */
+	public String toStringWithHiddenTokens() {
+		StringBuffer buf = new StringBuffer();
+		if ( hiddenTokens!=null ) {
+			for (int i = 0; i < hiddenTokens.size(); i++) {
+				Token hidden = (Token) hiddenTokens.get(i);
+				buf.append(hidden.getText());
+			}
+		}
+		String nodeText = this.toString();
+		if ( !nodeText.equals("<EOF>") ) buf.append(nodeText);
+		return buf.toString();
+	}
+
+	/** Print out the leaves of this tree, which means printing original
+	 *  input back out.
+	 */
+	public String toInputString() {
+		StringBuffer buf = new StringBuffer();
+		_toStringLeaves(buf);
+		return buf.toString();
+	}
+
+	public void _toStringLeaves(StringBuffer buf) {
+		if ( payload instanceof Token ) { // leaf node token?
+			buf.append(this.toStringWithHiddenTokens());
+			return;
+		}
+		for (int i = 0; children!=null && i < children.size(); i++) {
+			ParseTree t = (ParseTree)children.get(i);
+			t._toStringLeaves(buf);
+		}
+	}
+}
diff --git a/src/org/antlr/runtime/tree/RewriteCardinalityException.java b/src/org/antlr/runtime/tree/RewriteCardinalityException.java
new file mode 100755
index 0000000..7f909cd
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteCardinalityException.java
@@ -0,0 +1,47 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+/** Base class for all exceptions thrown during AST rewrite construction.
+ *  This signifies a case where the cardinality of two or more elements
+ *  in a subrule are different: (ID INT)+ where |ID|!=|INT|
+ */
+public class RewriteCardinalityException extends RuntimeException {
+	public String elementDescription;
+
+	public RewriteCardinalityException(String elementDescription) {
+		this.elementDescription = elementDescription;
+	}
+
+	public String getMessage() {
+		if ( elementDescription!=null ) {
+			return elementDescription;
+		}
+		return null;
+	}
+}
diff --git a/src/org/antlr/runtime/tree/RewriteEarlyExitException.java b/src/org/antlr/runtime/tree/RewriteEarlyExitException.java
new file mode 100755
index 0000000..9dbcc76
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteEarlyExitException.java
@@ -0,0 +1,39 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+/** No elements within a (...)+ in a rewrite rule */
+public class RewriteEarlyExitException extends RewriteCardinalityException {
+	public RewriteEarlyExitException() {
+		super(null);
+	}
+	public RewriteEarlyExitException(String elementDescription) {
+		super(elementDescription);
+	}
+
+}
diff --git a/src/org/antlr/runtime/tree/RewriteEmptyStreamException.java b/src/org/antlr/runtime/tree/RewriteEmptyStreamException.java
new file mode 100755
index 0000000..8f3a3fc
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteEmptyStreamException.java
@@ -0,0 +1,35 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+/** Ref to ID or expr but no tokens in ID stream or subtrees in expr stream */
+public class RewriteEmptyStreamException extends RewriteCardinalityException {
+	public RewriteEmptyStreamException(String elementDescription) {
+		super(elementDescription);
+	}
+}
diff --git a/src/org/antlr/runtime/tree/RewriteRuleElementStream.java b/src/org/antlr/runtime/tree/RewriteRuleElementStream.java
new file mode 100755
index 0000000..61f1860
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteRuleElementStream.java
@@ -0,0 +1,210 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** A generic list of elements tracked in an alternative to be used in
+ *  a -> rewrite rule.  We need to subclass to fill in the next() method,
+ *  which returns either an AST node wrapped around a token payload or
+ *  an existing subtree.
+ *
+ *  Once you start next()ing, do not try to add more elements.  It will
+ *  break the cursor tracking I believe.
+ *
+ *  @see org.antlr.runtime.tree.RewriteRuleSubtreeStream
+ *  @see org.antlr.runtime.tree.RewriteRuleTokenStream
+ *
+ *  TODO: add mechanism to detect/puke on modification after reading from stream
+ */
+public abstract class RewriteRuleElementStream {
+	/** Cursor 0..n-1.  If singleElement!=null, cursor is 0 until you next(),
+	 *  which bumps it to 1 meaning no more elements.
+	 */
+	protected int cursor = 0;
+
+	/** Track single elements w/o creating a list.  Upon 2nd add, alloc list */
+	protected Object singleElement;
+
+	/** The list of tokens or subtrees we are tracking */
+	protected List elements;
+
+	/** Once a node / subtree has been used in a stream, it must be dup'd
+	 *  from then on.  Streams are reset after subrules so that the streams
+	 *  can be reused in future subrules.  So, reset must set a dirty bit.
+	 *  If dirty, then next() always returns a dup.
+	 *
+	 *  I wanted to use "naughty bit" here, but couldn't think of a way
+	 *  to use "naughty".
+	 *
+	 *  TODO: unused?
+	 */
+	protected boolean dirty = false;
+
+	/** The element or stream description; usually has name of the token or
+	 *  rule reference that this list tracks.  Can include rulename too, but
+	 *  the exception would track that info.
+	 */
+	protected String elementDescription;
+	protected TreeAdaptor adaptor;
+
+	public RewriteRuleElementStream(TreeAdaptor adaptor, String elementDescription) {
+		this.elementDescription = elementDescription;
+		this.adaptor = adaptor;
+	}
+
+	/** Create a stream with one element */
+	public RewriteRuleElementStream(TreeAdaptor adaptor,
+									String elementDescription,
+									Object oneElement)
+	{
+		this(adaptor, elementDescription);
+		add(oneElement);
+	}
+
+	/** Create a stream, but feed off an existing list */
+	public RewriteRuleElementStream(TreeAdaptor adaptor,
+									String elementDescription,
+									List elements)
+	{
+		this(adaptor, elementDescription);
+		this.singleElement = null;
+		this.elements = elements;
+	}
+
+	/** Reset the condition of this stream so that it appears we have
+	 *  not consumed any of its elements.  Elements themselves are untouched.
+	 *  Once we reset the stream, any future use will need duplicates.  Set
+	 *  the dirty bit.
+	 */
+	public void reset() {
+		cursor = 0;
+		dirty = true;
+	}
+
+	public void add(Object el) {
+		//System.out.println("add '"+elementDescription+"' is "+el);
+		if ( el==null ) {
+			return;
+		}
+		if ( elements!=null ) { // if in list, just add
+			elements.add(el);
+			return;
+		}
+		if ( singleElement == null ) { // no elements yet, track w/o list
+			singleElement = el;
+			return;
+		}
+		// adding 2nd element, move to list
+		elements = new ArrayList(5);
+		elements.add(singleElement);
+		singleElement = null;
+		elements.add(el);
+	}
+
+	/** Return the next element in the stream.  If out of elements, throw
+	 *  an exception unless size()==1.  If size is 1, then return elements[0].
+	 *  Return a duplicate node/subtree if stream is out of elements and
+	 *  size==1.  If we've already used the element, dup (dirty bit set).
+	 */
+	public Object nextTree() {
+		int n = size();
+		if ( dirty || (cursor>=n && n==1) ) {
+			// if out of elements and size is 1, dup
+			Object el = _next();
+			return dup(el);
+		}
+		// test size above then fetch
+		Object el = _next();
+		return el;
+	}
+
+	/** do the work of getting the next element, making sure that it's
+	 *  a tree node or subtree.  Deal with the optimization of single-
+	 *  element list versus list of size > 1.  Throw an exception
+	 *  if the stream is empty or we're out of elements and size>1.
+	 *  protected so you can override in a subclass if necessary.
+	 */
+	protected Object _next() {
+		int n = size();
+		if ( n ==0 ) {
+			throw new RewriteEmptyStreamException(elementDescription);
+		}
+		if ( cursor>= n) { // out of elements?
+			if ( n ==1 ) {  // if size is 1, it's ok; return and we'll dup
+				return toTree(singleElement);
+			}
+			// out of elements and size was not 1, so we can't dup
+			throw new RewriteCardinalityException(elementDescription);
+		}
+		// we have elements
+		if ( singleElement!=null ) {
+			cursor++; // move cursor even for single element list
+			return toTree(singleElement);
+		}
+		// must have more than one in list, pull from elements
+		Object o = toTree(elements.get(cursor));
+		cursor++;
+		return o;
+	}
+
+	/** When constructing trees, sometimes we need to dup a token or AST
+	 * 	subtree.  Dup'ing a token means just creating another AST node
+	 *  around it.  For trees, you must call the adaptor.dupTree() unless
+	 *  the element is for a tree root; then it must be a node dup.
+	 */
+	protected abstract Object dup(Object el);
+
+	/** Ensure stream emits trees; tokens must be converted to AST nodes.
+	 *  AST nodes can be passed through unmolested.
+	 */
+	protected Object toTree(Object el) {
+		return el;
+	}
+
+	public boolean hasNext() {
+		 return (singleElement != null && cursor < 1) ||
+			   (elements!=null && cursor < elements.size());
+	}
+
+	public int size() {
+		int n = 0;
+		if ( singleElement != null ) {
+			n = 1;
+		}
+		if ( elements!=null ) {
+			return elements.size();
+		}
+		return n;
+	}
+
+	public String getDescription() {
+		return elementDescription;
+	}
+}
diff --git a/src/org/antlr/runtime/tree/RewriteRuleNodeStream.java b/src/org/antlr/runtime/tree/RewriteRuleNodeStream.java
new file mode 100755
index 0000000..713e9ff
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteRuleNodeStream.java
@@ -0,0 +1,70 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import java.util.List;
+
+/** Queues up nodes matched on left side of -> in a tree parser. This is
+ *  the analog of RewriteRuleTokenStream for normal parsers. 
+ */
+public class RewriteRuleNodeStream extends RewriteRuleElementStream {
+
+	public RewriteRuleNodeStream(TreeAdaptor adaptor, String elementDescription) {
+		super(adaptor, elementDescription);
+	}
+
+	/** Create a stream with one element */
+	public RewriteRuleNodeStream(TreeAdaptor adaptor,
+								 String elementDescription,
+								 Object oneElement)
+	{
+		super(adaptor, elementDescription, oneElement);
+	}
+
+	/** Create a stream, but feed off an existing list */
+	public RewriteRuleNodeStream(TreeAdaptor adaptor,
+								 String elementDescription,
+								 List elements)
+	{
+		super(adaptor, elementDescription, elements);
+	}
+
+	public Object nextNode() {
+		return _next();
+	}
+
+	protected Object toTree(Object el) {
+		return adaptor.dupNode(el);
+	}
+
+	protected Object dup(Object el) {
+		// we dup every node, so don't have to worry about calling dup; short-
+		// circuited next() so it doesn't call.
+		throw new UnsupportedOperationException("dup can't be called for a node stream.");
+	}
+}
diff --git a/src/org/antlr/runtime/tree/RewriteRuleSubtreeStream.java b/src/org/antlr/runtime/tree/RewriteRuleSubtreeStream.java
new file mode 100755
index 0000000..5189f21
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteRuleSubtreeStream.java
@@ -0,0 +1,88 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import java.util.List;
+
+public class RewriteRuleSubtreeStream extends RewriteRuleElementStream {
+
+	public RewriteRuleSubtreeStream(TreeAdaptor adaptor, String elementDescription) {
+		super(adaptor, elementDescription);
+	}
+
+	/** Create a stream with one element */
+	public RewriteRuleSubtreeStream(TreeAdaptor adaptor,
+									String elementDescription,
+									Object oneElement)
+	{
+		super(adaptor, elementDescription, oneElement);
+	}
+
+	/** Create a stream, but feed off an existing list */
+	public RewriteRuleSubtreeStream(TreeAdaptor adaptor,
+									String elementDescription,
+									List elements)
+	{
+		super(adaptor, elementDescription, elements);
+	}
+
+	/** Treat next element as a single node even if it's a subtree.
+	 *  This is used instead of next() when the result has to be a
+	 *  tree root node.  Also prevents us from duplicating recently-added
+	 *  children; e.g., ^(type ID)+ adds ID to type and then 2nd iteration
+	 *  must dup the type node, but ID has been added.
+	 *
+	 *  Referencing a rule result twice is ok; dup entire tree as
+	 *  we can't be adding trees as root; e.g., expr expr.
+	 *
+	 *  Hideous code duplication here with super.next().  Can't think of
+	 *  a proper way to refactor.  This needs to always call dup node
+	 *  and super.next() doesn't know which to call: dup node or dup tree.
+	 */
+	public Object nextNode() {
+		//System.out.println("nextNode: elements="+elements+", singleElement="+((Tree)singleElement).toStringTree());
+		int n = size();
+		if ( dirty || (cursor>=n && n==1) ) {
+			// if out of elements and size is 1, dup (at most a single node
+			// since this is for making root nodes).
+			Object el = _next();
+			return adaptor.dupNode(el);
+		}
+		// test size above then fetch
+		Object tree = _next();
+		while (adaptor.isNil(tree) && adaptor.getChildCount(tree) == 1)
+			tree = adaptor.getChild(tree, 0);
+		//System.out.println("_next="+((Tree)tree).toStringTree());
+		Object el = adaptor.dupNode(tree); // dup just the root (want node here)
+		return el;
+	}
+
+	protected Object dup(Object el) {
+		return adaptor.dupTree(el);
+	}
+}
\ No newline at end of file
diff --git a/src/org/antlr/runtime/tree/RewriteRuleTokenStream.java b/src/org/antlr/runtime/tree/RewriteRuleTokenStream.java
new file mode 100755
index 0000000..4cd7b08
--- /dev/null
+++ b/src/org/antlr/runtime/tree/RewriteRuleTokenStream.java
@@ -0,0 +1,76 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+
+import java.util.List;
+
+public class RewriteRuleTokenStream extends RewriteRuleElementStream {
+
+	public RewriteRuleTokenStream(TreeAdaptor adaptor, String elementDescription) {
+		super(adaptor, elementDescription);
+	}
+
+	/** Create a stream with one element */
+	public RewriteRuleTokenStream(TreeAdaptor adaptor,
+								  String elementDescription,
+								  Object oneElement)
+	{
+		super(adaptor, elementDescription, oneElement);
+	}
+
+	/** Create a stream, but feed off an existing list */
+	public RewriteRuleTokenStream(TreeAdaptor adaptor,
+								  String elementDescription,
+								  List elements)
+	{
+		super(adaptor, elementDescription, elements);
+	}
+
+	/** Get next token from stream and make a node for it */
+	public Object nextNode() {
+		Token t = (Token)_next();
+		return adaptor.create(t);
+	}
+
+	public Token nextToken() {
+		return (Token)_next();
+	}
+
+	/** Don't convert to a tree unless they explicitly call nextTree.
+	 *  This way we can do hetero tree nodes in rewrite.
+	 */
+	protected Object toTree(Object el) {
+		return el;
+	}
+
+	protected Object dup(Object el) {
+		throw new UnsupportedOperationException("dup can't be called for a token stream.");
+	}
+}
diff --git a/src/org/antlr/runtime/tree/Tree.java b/src/org/antlr/runtime/tree/Tree.java
new file mode 100755
index 0000000..7875be3
--- /dev/null
+++ b/src/org/antlr/runtime/tree/Tree.java
@@ -0,0 +1,127 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+
+import java.util.List;
+
+/** What does a tree look like?  ANTLR has a number of support classes
+ *  such as CommonTreeNodeStream that work on these kinds of trees.  You
+ *  don't have to make your trees implement this interface, but if you do,
+ *  you'll be able to use more support code.
+ *
+ *  NOTE: When constructing trees, ANTLR can build any kind of tree; it can
+ *  even use Token objects as trees if you add a child list to your tokens.
+ *
+ *  This is a tree node without any payload; just navigation and factory stuff.
+ */
+public interface Tree {
+	public static final Tree INVALID_NODE = new CommonTree(Token.INVALID_TOKEN);
+
+	Tree getChild(int i);
+
+	int getChildCount();
+
+	// Tree tracks parent and child index now > 3.0
+
+	public Tree getParent();
+
+	public void setParent(Tree t);
+
+    /** Is there is a node above with token type ttype? */
+    public boolean hasAncestor(int ttype);
+
+    /** Walk upwards and get first ancestor with this token type. */
+    public Tree getAncestor(int ttype);
+
+    /** Return a list of all ancestors of this node.  The first node of
+     *  list is the root and the last is the parent of this node.
+     */
+    public List getAncestors();
+
+    /** This node is what child index? 0..n-1 */
+	public int getChildIndex();
+
+	public void setChildIndex(int index);
+
+	/** Set the parent and child index values for all children */
+	public void freshenParentAndChildIndexes();
+
+	/** Add t as a child to this node.  If t is null, do nothing.  If t
+	 *  is nil, add all children of t to this' children.
+	 */
+	void addChild(Tree t);
+
+	/** Set ith child (0..n-1) to t; t must be non-null and non-nil node */
+	public void setChild(int i, Tree t);
+
+	public Object deleteChild(int i);
+
+	/** Delete children from start to stop and replace with t even if t is
+	 *  a list (nil-root tree).  num of children can increase or decrease.
+	 *  For huge child lists, inserting children can force walking rest of
+	 *  children to set their childindex; could be slow.
+	 */
+	public void replaceChildren(int startChildIndex, int stopChildIndex, Object t);	
+
+	/** Indicates the node is a nil node but may still have children, meaning
+	 *  the tree is a flat list.
+	 */
+	boolean isNil();
+
+	/**  What is the smallest token index (indexing from 0) for this node
+	 *   and its children?
+	 */
+	int getTokenStartIndex();
+
+	void setTokenStartIndex(int index);
+
+	/**  What is the largest token index (indexing from 0) for this node
+	 *   and its children?
+	 */
+	int getTokenStopIndex();
+
+	void setTokenStopIndex(int index);
+
+	Tree dupNode();
+
+	/** Return a token type; needed for tree parsing */
+	int getType();
+
+	String getText();
+
+	/** In case we don't have a token payload, what is the line for errors? */
+	int getLine();
+
+	int getCharPositionInLine();
+
+	String toStringTree();
+
+	String toString();
+}
diff --git a/src/org/antlr/runtime/tree/TreeAdaptor.java b/src/org/antlr/runtime/tree/TreeAdaptor.java
new file mode 100755
index 0000000..8e889f8
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeAdaptor.java
@@ -0,0 +1,263 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.RecognitionException;
+
+/** How to create and navigate trees.  Rather than have a separate factory
+ *  and adaptor, I've merged them.  Makes sense to encapsulate.
+ *
+ *  This takes the place of the tree construction code generated in the
+ *  generated code in 2.x and the ASTFactory.
+ *
+ *  I do not need to know the type of a tree at all so they are all
+ *  generic Objects.  This may increase the amount of typecasting needed. :(
+ */
+public interface TreeAdaptor {
+	// C o n s t r u c t i o n
+
+	/** Create a tree node from Token object; for CommonTree type trees,
+	 *  then the token just becomes the payload.  This is the most
+	 *  common create call.
+	 *
+	 *  Override if you want another kind of node to be built.
+	 */
+	public Object create(Token payload);
+
+	/** Duplicate a single tree node.
+	 *  Override if you want another kind of node to be built.
+	 */
+	public Object dupNode(Object treeNode);
+
+	/** Duplicate tree recursively, using dupNode() for each node */
+	public Object dupTree(Object tree);
+
+	/** Return a nil node (an empty but non-null node) that can hold
+	 *  a list of element as the children.  If you want a flat tree (a list)
+	 *  use "t=adaptor.nil(); t.addChild(x); t.addChild(y);"
+	 */
+	public Object nil();
+
+	/** Return a tree node representing an error.  This node records the
+	 *  tokens consumed during error recovery.  The start token indicates the
+	 *  input symbol at which the error was detected.  The stop token indicates
+	 *  the last symbol consumed during recovery.
+	 *
+	 *  You must specify the input stream so that the erroneous text can
+	 *  be packaged up in the error node.  The exception could be useful
+	 *  to some applications; default implementation stores ptr to it in
+	 *  the CommonErrorNode.
+	 *
+	 *  This only makes sense during token parsing, not tree parsing.
+	 *  Tree parsing should happen only when parsing and tree construction
+	 *  succeed.
+	 */
+	public Object errorNode(TokenStream input, Token start, Token stop, RecognitionException e);
+
+	/** Is tree considered a nil node used to make lists of child nodes? */
+	public boolean isNil(Object tree);
+
+	/** Add a child to the tree t.  If child is a flat tree (a list), make all
+	 *  in list children of t.  Warning: if t has no children, but child does
+	 *  and child isNil then you can decide it is ok to move children to t via
+	 *  t.children = child.children; i.e., without copying the array.  Just
+	 *  make sure that this is consistent with have the user will build
+	 *  ASTs.  Do nothing if t or child is null.
+	 */
+	public void addChild(Object t, Object child);
+
+	/** If oldRoot is a nil root, just copy or move the children to newRoot.
+	 *  If not a nil root, make oldRoot a child of newRoot.
+	 *
+	 *    old=^(nil a b c), new=r yields ^(r a b c)
+	 *    old=^(a b c), new=r yields ^(r ^(a b c))
+	 *
+	 *  If newRoot is a nil-rooted single child tree, use the single
+	 *  child as the new root node.
+	 *
+	 *    old=^(nil a b c), new=^(nil r) yields ^(r a b c)
+	 *    old=^(a b c), new=^(nil r) yields ^(r ^(a b c))
+	 *
+	 *  If oldRoot was null, it's ok, just return newRoot (even if isNil).
+	 *
+	 *    old=null, new=r yields r
+	 *    old=null, new=^(nil r) yields ^(nil r)
+	 *
+	 *  Return newRoot.  Throw an exception if newRoot is not a
+	 *  simple node or nil root with a single child node--it must be a root
+	 *  node.  If newRoot is ^(nil x) return x as newRoot.
+	 *
+	 *  Be advised that it's ok for newRoot to point at oldRoot's
+	 *  children; i.e., you don't have to copy the list.  We are
+	 *  constructing these nodes so we should have this control for
+	 *  efficiency.
+	 */
+	public Object becomeRoot(Object newRoot, Object oldRoot);
+
+	/** Given the root of the subtree created for this rule, post process
+	 *  it to do any simplifications or whatever you want.  A required
+	 *  behavior is to convert ^(nil singleSubtree) to singleSubtree
+	 *  as the setting of start/stop indexes relies on a single non-nil root
+	 *  for non-flat trees.
+	 *
+	 *  Flat trees such as for lists like "idlist : ID+ ;" are left alone
+	 *  unless there is only one ID.  For a list, the start/stop indexes
+	 *  are set in the nil node.
+	 *
+	 *  This method is executed after all rule tree construction and right
+	 *  before setTokenBoundaries().
+	 */
+	public Object rulePostProcessing(Object root);
+
+	/** For identifying trees.
+	 *
+	 *  How to identify nodes so we can say "add node to a prior node"?
+	 *  Even becomeRoot is an issue.  Use System.identityHashCode(node)
+	 *  usually.
+	 */
+	public int getUniqueID(Object node);
+
+
+	// R e w r i t e  R u l e s
+
+	/** Create a node for newRoot make it the root of oldRoot.
+	 *  If oldRoot is a nil root, just copy or move the children to newRoot.
+	 *  If not a nil root, make oldRoot a child of newRoot.
+	 *
+	 *  Return node created for newRoot.
+	 *
+	 *  Be advised: when debugging ASTs, the DebugTreeAdaptor manually
+	 *  calls create(Token child) and then plain becomeRoot(node, node)
+	 *  because it needs to trap calls to create, but it can't since it delegates
+	 *  to not inherits from the TreeAdaptor.
+	 */
+	public Object becomeRoot(Token newRoot, Object oldRoot);
+
+	/** Create a new node derived from a token, with a new token type.
+	 *  This is invoked from an imaginary node ref on right side of a
+	 *  rewrite rule as IMAG[$tokenLabel].
+	 *
+	 *  This should invoke createToken(Token).
+	 */
+	public Object create(int tokenType, Token fromToken);
+
+	/** Same as create(tokenType,fromToken) except set the text too.
+	 *  This is invoked from an imaginary node ref on right side of a
+	 *  rewrite rule as IMAG[$tokenLabel, "IMAG"].
+	 *
+	 *  This should invoke createToken(Token).
+	 */
+	public Object create(int tokenType, Token fromToken, String text);
+
+	/** Create a new node derived from a token, with a new token type.
+	 *  This is invoked from an imaginary node ref on right side of a
+	 *  rewrite rule as IMAG["IMAG"].
+	 *
+	 *  This should invoke createToken(int,String).
+	 */
+	public Object create(int tokenType, String text);
+
+
+	// C o n t e n t
+
+	/** For tree parsing, I need to know the token type of a node */
+	public int getType(Object t);
+
+	/** Node constructors can set the type of a node */
+	public void setType(Object t, int type);
+
+	public String getText(Object t);
+
+	/** Node constructors can set the text of a node */
+	public void setText(Object t, String text);
+
+	/** Return the token object from which this node was created.
+	 *  Currently used only for printing an error message.
+	 *  The error display routine in BaseRecognizer needs to
+	 *  display where the input the error occurred. If your
+	 *  tree of limitation does not store information that can
+	 *  lead you to the token, you can create a token filled with
+	 *  the appropriate information and pass that back.  See
+	 *  BaseRecognizer.getErrorMessage().
+	 */
+	public Token getToken(Object t);
+
+	/** Where are the bounds in the input token stream for this node and
+	 *  all children?  Each rule that creates AST nodes will call this
+	 *  method right before returning.  Flat trees (i.e., lists) will
+	 *  still usually have a nil root node just to hold the children list.
+	 *  That node would contain the start/stop indexes then.
+	 */
+	public void setTokenBoundaries(Object t, Token startToken, Token stopToken);
+
+	/** Get the token start index for this subtree; return -1 if no such index */
+	public int getTokenStartIndex(Object t);
+
+	/** Get the token stop index for this subtree; return -1 if no such index */
+	public int getTokenStopIndex(Object t);
+
+
+	// N a v i g a t i o n  /  T r e e  P a r s i n g
+
+	/** Get a child 0..n-1 node */
+	public Object getChild(Object t, int i);
+
+	/** Set ith child (0..n-1) to t; t must be non-null and non-nil node */
+	public void setChild(Object t, int i, Object child);
+
+	/** Remove ith child and shift children down from right. */
+	public Object deleteChild(Object t, int i);
+
+	/** How many children?  If 0, then this is a leaf node */
+	public int getChildCount(Object t);
+
+	/** Who is the parent node of this node; if null, implies node is root.
+	 *  If your node type doesn't handle this, it's ok but the tree rewrites
+	 *  in tree parsers need this functionality.
+	 */
+	public Object getParent(Object t);
+	public void setParent(Object t, Object parent);
+
+	/** What index is this node in the child list? Range: 0..n-1
+	 *  If your node type doesn't handle this, it's ok but the tree rewrites
+	 *  in tree parsers need this functionality.
+	 */
+	public int getChildIndex(Object t);
+	public void setChildIndex(Object t, int index);
+
+	/** Replace from start to stop child index of parent with t, which might
+	 *  be a list.  Number of children may be different
+	 *  after this call.
+	 *
+	 *  If parent is null, don't do anything; must be at root of overall tree.
+	 *  Can't replace whatever points to the parent externally.  Do nothing.
+	 */
+	public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t);
+}
diff --git a/src/org/antlr/runtime/tree/TreeFilter.java b/src/org/antlr/runtime/tree/TreeFilter.java
new file mode 100755
index 0000000..b6a7e05
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeFilter.java
@@ -0,0 +1,135 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.RecognizerSharedState;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.TokenStream;
+
+/**
+ Cut-n-paste from material I'm not using in the book anymore (edit later
+ to make sense):
+
+ Now, how are we going to test these tree patterns against every
+subtree in our original tree?  In what order should we visit nodes?
+For this application, it turns out we need a simple ``apply once''
+rule application strategy and a ``down then up'' tree traversal
+strategy.  Let's look at rule application first.
+
+As we visit each node, we need to see if any of our patterns match. If
+a pattern matches, we execute the associated tree rewrite and move on
+to the next node. In other words, we only look for a single rule
+application opportunity (we'll see below that we sometimes need to
+repeatedly apply rules). The following method applies a rule in a @cl
+TreeParser (derived from a tree grammar) to a tree:
+
+here is where weReferenced code/walking/patterns/TreePatternMatcher.java
+
+It uses reflection to lookup the appropriate rule within the generated
+tree parser class (@cl Simplify in this case). Most of the time, the
+rule will not match the tree.  To avoid issuing syntax errors and
+attempting error recovery, it bumps up the backtracking level.  Upon
+failure, the invoked rule immediately returns. If you don't plan on
+using this technique in your own ANTLR-based application, don't sweat
+the details. This method boils down to ``call a rule to match a tree,
+executing any embedded actions and rewrite rules.''
+
+At this point, we know how to define tree grammar rules and how to
+apply them to a particular subtree. The final piece of the tree
+pattern matcher is the actual tree traversal. We have to get the
+correct node visitation order.  In particular, we need to perform the
+scalar-vector multiply transformation on the way down (preorder) and
+we need to reduce multiply-by-zero subtrees on the way up (postorder).
+
+To implement a top-down visitor, we do a depth first walk of the tree,
+executing an action in the preorder position. To get a bottom-up
+visitor, we execute an action in the postorder position.  ANTLR
+provides a standard @cl TreeVisitor class with a depth first search @v
+visit method. That method executes either a @m pre or @m post method
+or both. In our case, we need to call @m applyOnce in both. On the way
+down, we'll look for @r vmult patterns. On the way up,
+we'll look for @r mult0 patterns.
+ */
+public class TreeFilter extends TreeParser {
+    public interface fptr {
+        public void rule() throws RecognitionException;
+    }
+
+    protected TokenStream originalTokenStream;
+    protected TreeAdaptor originalAdaptor;
+
+    public TreeFilter(TreeNodeStream input) {
+        this(input, new RecognizerSharedState());
+    }
+    public TreeFilter(TreeNodeStream input, RecognizerSharedState state) {
+        super(input, state);
+        originalAdaptor = input.getTreeAdaptor();
+        originalTokenStream = input.getTokenStream();
+    }
+
+    public void applyOnce(Object t, fptr whichRule) {
+        if ( t==null ) return;
+        try {
+            // share TreeParser object but not parsing-related state
+            state = new RecognizerSharedState();
+            input = new CommonTreeNodeStream(originalAdaptor, t);
+            ((CommonTreeNodeStream)input).setTokenStream(originalTokenStream);
+            setBacktrackingLevel(1);
+            whichRule.rule();
+            setBacktrackingLevel(0);
+        }
+        catch (RecognitionException e) { ; }
+    }
+
+    public void downup(Object t) {
+        TreeVisitor v = new TreeVisitor(new CommonTreeAdaptor());
+        TreeVisitorAction actions = new TreeVisitorAction() {
+            public Object pre(Object t)  { applyOnce(t, topdown_fptr); return t; }
+            public Object post(Object t) { applyOnce(t, bottomup_fptr); return t; }
+        };
+        v.visit(t, actions);
+    }
+        
+    fptr topdown_fptr = new fptr() {
+        public void rule() throws RecognitionException {
+            topdown();
+        }
+    };
+
+    fptr bottomup_fptr = new fptr() {
+        public void rule() throws RecognitionException {
+            bottomup();
+        }
+    };
+
+    // methods the downup strategy uses to do the up and down rules.
+    // to override, just define tree grammar rule topdown and turn on
+    // filter=true.
+    public void topdown() throws RecognitionException {;}
+    public void bottomup() throws RecognitionException {;}
+}
diff --git a/src/org/antlr/runtime/tree/TreeIterator.java b/src/org/antlr/runtime/tree/TreeIterator.java
new file mode 100755
index 0000000..43ead6d
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeIterator.java
@@ -0,0 +1,132 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.CommonToken;
+import org.antlr.runtime.misc.FastQueue;
+
+import java.util.Iterator;
+
+/** Return a node stream from a doubly-linked tree whose nodes
+ *  know what child index they are.  No remove() is supported.
+ *
+ *  Emit navigation nodes (DOWN, UP, and EOF) to let show tree structure.
+ */
+public class TreeIterator implements Iterator {
+    protected TreeAdaptor adaptor;
+    protected Object root;
+    protected Object tree;
+    protected boolean firstTime = true;
+
+    // navigation nodes to return during walk and at end
+    public Object up;
+    public Object down;
+    public Object eof;
+
+    /** If we emit UP/DOWN nodes, we need to spit out multiple nodes per
+     *  next() call.
+     */
+    protected FastQueue nodes;
+
+    public TreeIterator(Object tree) {
+        this(new CommonTreeAdaptor(),tree);
+    }
+
+    public TreeIterator(TreeAdaptor adaptor, Object tree) {
+        this.adaptor = adaptor;
+        this.tree = tree;
+        this.root = tree;
+        nodes = new FastQueue();
+        down = adaptor.create(Token.DOWN, "DOWN");
+        up = adaptor.create(Token.UP, "UP");
+        eof = adaptor.create(Token.EOF, "EOF");
+    }
+
+    public void reset() {
+        firstTime = true;
+        tree = root;
+        nodes.clear();
+    }
+
+    public boolean hasNext() {
+        if ( firstTime ) return root!=null;
+        if ( nodes!=null && nodes.size()>0 ) return true;
+        if ( tree==null ) return false;
+        if ( adaptor.getChildCount(tree)>0 ) return true;
+        return adaptor.getParent(tree)!=null; // back at root?
+    }
+
+    public Object next() {
+        if ( firstTime ) { // initial condition
+            firstTime = false;
+            if ( adaptor.getChildCount(tree)==0 ) { // single node tree (special)
+                nodes.add(eof);
+                return tree;
+            }
+            return tree;
+        }
+        // if any queued up, use those first
+        if ( nodes!=null && nodes.size()>0 ) return nodes.remove();
+
+        // no nodes left?
+        if ( tree==null ) return eof;
+
+        // next node will be child 0 if any children
+        if ( adaptor.getChildCount(tree)>0 ) {
+            tree = adaptor.getChild(tree, 0);
+            nodes.add(tree); // real node is next after DOWN
+            return down;
+        }
+        // if no children, look for next sibling of tree or ancestor
+        Object parent = adaptor.getParent(tree);
+        // while we're out of siblings, keep popping back up towards root
+        while ( parent!=null &&
+                adaptor.getChildIndex(tree)+1 >= adaptor.getChildCount(parent) )
+        {
+            nodes.add(up); // we're moving back up
+            tree = parent;
+            parent = adaptor.getParent(tree);
+        }
+        // no nodes left?
+        if ( parent==null ) {
+            tree = null; // back at root? nothing left then
+            nodes.add(eof); // add to queue, might have UP nodes in there
+            return nodes.remove();
+        }
+
+        // must have found a node with an unvisited sibling
+        // move to it and return it
+        int nextSiblingIndex = adaptor.getChildIndex(tree) + 1;
+        tree = adaptor.getChild(parent, nextSiblingIndex);
+        nodes.add(tree); // add to queue, might have UP nodes in there
+        return nodes.remove();
+    }
+
+    public void remove() { throw new UnsupportedOperationException(); }
+}
diff --git a/src/org/antlr/runtime/tree/TreeNodeStream.java b/src/org/antlr/runtime/tree/TreeNodeStream.java
new file mode 100755
index 0000000..df0ad34
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeNodeStream.java
@@ -0,0 +1,106 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.IntStream;
+import org.antlr.runtime.TokenStream;
+
+/** A stream of tree nodes, accessing nodes from a tree of some kind */
+public interface TreeNodeStream extends IntStream {
+	/** Get a tree node at an absolute index i; 0..n-1.
+	 *  If you don't want to buffer up nodes, then this method makes no
+	 *  sense for you.
+	 */
+	public Object get(int i);
+
+	/** Get tree node at current input pointer + i ahead where i=1 is next node.
+	 *  i<0 indicates nodes in the past.  So LT(-1) is previous node, but
+	 *  implementations are not required to provide results for k < -1.
+	 *  LT(0) is undefined.  For i>=n, return null.
+	 *  Return null for LT(0) and any index that results in an absolute address
+	 *  that is negative.
+	 *
+	 *  This is analogus to the LT() method of the TokenStream, but this
+	 *  returns a tree node instead of a token.  Makes code gen identical
+	 *  for both parser and tree grammars. :)
+	 */
+	public Object LT(int k);
+
+	/** Where is this stream pulling nodes from?  This is not the name, but
+	 *  the object that provides node objects.
+	 */
+	public Object getTreeSource();
+
+	/** If the tree associated with this stream was created from a TokenStream,
+	 *  you can specify it here.  Used to do rule $text attribute in tree
+	 *  parser.  Optional unless you use tree parser rule text attribute
+	 *  or output=template and rewrite=true options.
+	 */
+	public TokenStream getTokenStream();
+
+	/** What adaptor can tell me how to interpret/navigate nodes and
+	 *  trees.  E.g., get text of a node.
+	 */
+	public TreeAdaptor getTreeAdaptor();
+
+	/** As we flatten the tree, we use UP, DOWN nodes to represent
+	 *  the tree structure.  When debugging we need unique nodes
+	 *  so we have to instantiate new ones.  When doing normal tree
+	 *  parsing, it's slow and a waste of memory to create unique
+	 *  navigation nodes.  Default should be false;
+	 */
+	public void setUniqueNavigationNodes(boolean uniqueNavigationNodes);
+
+    /** Reset the tree node stream in such a way that it acts like
+     *  a freshly constructed stream.
+     */
+    public void reset();
+
+	/** Return the text of all nodes from start to stop, inclusive.
+	 *  If the stream does not buffer all the nodes then it can still
+	 *  walk recursively from start until stop.  You can always return
+	 *  null or "" too, but users should not access $ruleLabel.text in
+	 *  an action of course in that case.
+	 */
+	public String toString(Object start, Object stop);
+
+
+	// REWRITING TREES (used by tree parser)
+
+	/** Replace from start to stop child index of parent with t, which might
+	 *  be a list.  Number of children may be different
+	 *  after this call.  The stream is notified because it is walking the
+	 *  tree and might need to know you are monkeying with the underlying
+	 *  tree.  Also, it might be able to modify the node stream to avoid
+	 *  restreaming for future phases.
+	 *
+	 *  If parent is null, don't do anything; must be at root of overall tree.
+	 *  Can't replace whatever points to the parent externally.  Do nothing.
+	 */
+	public void replaceChildren(Object parent, int startChildIndex, int stopChildIndex, Object t);
+}
diff --git a/src/org/antlr/runtime/tree/TreeParser.java b/src/org/antlr/runtime/tree/TreeParser.java
new file mode 100755
index 0000000..e568bc9
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeParser.java
@@ -0,0 +1,169 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.*;
+
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+
+/** A parser for a stream of tree nodes.  "tree grammars" result in a subclass
+ *  of this.  All the error reporting and recovery is shared with Parser via
+ *  the BaseRecognizer superclass.
+*/
+public class TreeParser extends BaseRecognizer {
+	public static final int DOWN = Token.DOWN;
+	public static final int UP = Token.UP;
+
+    // precompiled regex used by inContext
+    static String dotdot = ".*[^.]\\.\\.[^.].*";
+    static String doubleEtc = ".*\\.\\.\\.\\s+\\.\\.\\..*";
+    static Pattern dotdotPattern = Pattern.compile(dotdot);
+    static Pattern doubleEtcPattern = Pattern.compile(doubleEtc);
+
+	protected TreeNodeStream input;
+
+	public TreeParser(TreeNodeStream input) {
+		super(); // highlight that we go to super to set state object
+		setTreeNodeStream(input);
+	}
+
+	public TreeParser(TreeNodeStream input, RecognizerSharedState state) {
+		super(state); // share the state object with another parser
+		setTreeNodeStream(input);
+    }
+
+	public void reset() {
+		super.reset(); // reset all recognizer state variables
+		if ( input!=null ) {
+			input.seek(0); // rewind the input
+		}
+	}
+
+	/** Set the input stream */
+	public void setTreeNodeStream(TreeNodeStream input) {
+		this.input = input;
+	}
+
+	public TreeNodeStream getTreeNodeStream() {
+		return input;
+	}
+
+	public String getSourceName() {
+		return input.getSourceName();
+	}
+
+	protected Object getCurrentInputSymbol(IntStream input) {
+		return ((TreeNodeStream)input).LT(1);
+	}
+
+	protected Object getMissingSymbol(IntStream input,
+									  RecognitionException e,
+									  int expectedTokenType,
+									  BitSet follow)
+	{
+		String tokenText =
+			"<missing "+getTokenNames()[expectedTokenType]+">";
+        TreeAdaptor adaptor = ((TreeNodeStream)e.input).getTreeAdaptor();
+        return adaptor.create(new CommonToken(expectedTokenType, tokenText));
+	}
+
+    /** Match '.' in tree parser has special meaning.  Skip node or
+	 *  entire tree if node has children.  If children, scan until
+	 *  corresponding UP node.
+	 */
+	public void matchAny(IntStream ignore) { // ignore stream, copy of input
+		state.errorRecovery = false;
+		state.failed = false;
+		Object look = input.LT(1);
+		if ( input.getTreeAdaptor().getChildCount(look)==0 ) {
+			input.consume(); // not subtree, consume 1 node and return
+			return;
+		}
+		// current node is a subtree, skip to corresponding UP.
+		// must count nesting level to get right UP
+		int level=0;
+		int tokenType = input.getTreeAdaptor().getType(look);
+		while ( tokenType!=Token.EOF && !(tokenType==UP && level==0) ) {
+			input.consume();
+			look = input.LT(1);
+			tokenType = input.getTreeAdaptor().getType(look);
+			if ( tokenType == DOWN ) {
+				level++;
+			}
+			else if ( tokenType == UP ) {
+				level--;
+			}
+		}
+		input.consume(); // consume UP
+	}
+
+    /** We have DOWN/UP nodes in the stream that have no line info; override.
+	 *  plus we want to alter the exception type.  Don't try to recover
+	 *  from tree parser errors inline...
+     */
+    protected Object recoverFromMismatchedToken(IntStream input,
+                                                int ttype,
+                                                BitSet follow)
+        throws RecognitionException
+    {
+        throw new MismatchedTreeNodeException(ttype, (TreeNodeStream)input);
+    }
+
+    /** Prefix error message with the grammar name because message is
+	 *  always intended for the programmer because the parser built
+	 *  the input tree not the user.
+	 */
+	public String getErrorHeader(RecognitionException e) {
+		return getGrammarFileName()+": node from "+
+			   (e.approximateLineInfo?"after ":"")+"line "+e.line+":"+e.charPositionInLine;
+	}
+
+	/** Tree parsers parse nodes they usually have a token object as
+	 *  payload. Set the exception token and do the default behavior.
+	 */
+	public String getErrorMessage(RecognitionException e, String[] tokenNames) {
+		if ( this instanceof TreeParser ) {
+			TreeAdaptor adaptor = ((TreeNodeStream)e.input).getTreeAdaptor();
+			e.token = adaptor.getToken(e.node);
+			if ( e.token==null ) { // could be an UP/DOWN node
+				e.token = new CommonToken(adaptor.getType(e.node),
+										  adaptor.getText(e.node));
+			}
+		}
+		return super.getErrorMessage(e, tokenNames);
+	}
+
+	public void traceIn(String ruleName, int ruleIndex)  {
+		super.traceIn(ruleName, ruleIndex, input.LT(1));
+	}
+
+	public void traceOut(String ruleName, int ruleIndex)  {
+		super.traceOut(ruleName, ruleIndex, input.LT(1));
+	}
+}
diff --git a/src/org/antlr/runtime/tree/TreePatternLexer.java b/src/org/antlr/runtime/tree/TreePatternLexer.java
new file mode 100755
index 0000000..2677c4e
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreePatternLexer.java
@@ -0,0 +1,135 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+public class TreePatternLexer {
+	public static final int EOF = -1;
+	public static final int BEGIN = 1;
+	public static final int END = 2;
+	public static final int ID = 3;
+	public static final int ARG = 4;
+	public static final int PERCENT = 5;
+	public static final int COLON = 6;
+	public static final int DOT = 7;
+
+	/** The tree pattern to lex like "(A B C)" */
+	protected String pattern;
+
+	/** Index into input string */
+	protected int p = -1;
+
+	/** Current char */
+	protected int c;
+
+	/** How long is the pattern in char? */
+	protected int n;
+
+	/** Set when token type is ID or ARG (name mimics Java's StreamTokenizer) */
+	public StringBuffer sval = new StringBuffer();
+
+	public boolean error = false;
+
+	public TreePatternLexer(String pattern) {
+		this.pattern = pattern;
+		this.n = pattern.length();
+		consume();
+	}
+
+	public int nextToken() {
+		sval.setLength(0); // reset, but reuse buffer
+		while ( c != EOF ) {
+			if ( c==' ' || c=='\n' || c=='\r' || c=='\t' ) {
+				consume();
+				continue;
+			}
+			if ( (c>='a' && c<='z') || (c>='A' && c<='Z') || c=='_' ) {
+				sval.append((char)c);
+				consume();
+				while ( (c>='a' && c<='z') || (c>='A' && c<='Z') ||
+						(c>='0' && c<='9') || c=='_' )
+				{
+					sval.append((char)c);
+					consume();
+				}
+				return ID;
+			}
+			if ( c=='(' ) {
+				consume();
+				return BEGIN;
+			}
+			if ( c==')' ) {
+				consume();
+				return END;
+			}
+			if ( c=='%' ) {
+				consume();
+				return PERCENT;
+			}
+			if ( c==':' ) {
+				consume();
+				return COLON;
+			}
+			if ( c=='.' ) {
+				consume();
+				return DOT;
+			}
+			if ( c=='[' ) { // grab [x] as a string, returning x
+				consume();
+				while ( c!=']' ) {
+					if ( c=='\\' ) {
+						consume();
+						if ( c!=']' ) {
+							sval.append('\\');
+						}
+						sval.append((char)c);
+					}
+					else {
+						sval.append((char)c);
+					}
+					consume();
+				}
+				consume();
+				return ARG;
+			}
+			consume();
+			error = true;
+			return EOF;
+		}
+		return EOF;
+	}
+
+	protected void consume() {
+		p++;
+		if ( p>=n ) {
+			c = EOF;
+		}
+		else {
+			c = pattern.charAt(p);
+		}
+	}
+}
\ No newline at end of file
diff --git a/src/org/antlr/runtime/tree/TreePatternParser.java b/src/org/antlr/runtime/tree/TreePatternParser.java
new file mode 100755
index 0000000..14983ab
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreePatternParser.java
@@ -0,0 +1,154 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+import org.antlr.runtime.CommonToken;
+
+public class TreePatternParser {
+	protected TreePatternLexer tokenizer;
+	protected int ttype;
+	protected TreeWizard wizard;
+	protected TreeAdaptor adaptor;
+
+	public TreePatternParser(TreePatternLexer tokenizer, TreeWizard wizard, TreeAdaptor adaptor) {
+		this.tokenizer = tokenizer;
+		this.wizard = wizard;
+		this.adaptor = adaptor;
+		ttype = tokenizer.nextToken(); // kickstart
+	}
+
+	public Object pattern() {
+		if ( ttype==TreePatternLexer.BEGIN ) {
+			return parseTree();
+		}
+		else if ( ttype==TreePatternLexer.ID ) {
+			Object node = parseNode();
+			if ( ttype==TreePatternLexer.EOF ) {
+				return node;
+			}
+			return null; // extra junk on end
+		}
+		return null;
+	}
+
+	public Object parseTree() {
+		if ( ttype != TreePatternLexer.BEGIN ) {
+			throw new RuntimeException("no BEGIN");
+		}
+		ttype = tokenizer.nextToken();
+		Object root = parseNode();
+		if ( root==null ) {
+			return null;
+		}
+		while ( ttype==TreePatternLexer.BEGIN ||
+				ttype==TreePatternLexer.ID ||
+				ttype==TreePatternLexer.PERCENT ||
+				ttype==TreePatternLexer.DOT )
+		{
+			if ( ttype==TreePatternLexer.BEGIN ) {
+				Object subtree = parseTree();
+				adaptor.addChild(root, subtree);
+			}
+			else {
+				Object child = parseNode();
+				if ( child==null ) {
+					return null;
+				}
+				adaptor.addChild(root, child);
+			}
+		}
+		if ( ttype != TreePatternLexer.END ) {
+			throw new RuntimeException("no END");
+		}
+		ttype = tokenizer.nextToken();
+		return root;
+	}
+
+	public Object parseNode() {
+		// "%label:" prefix
+		String label = null;
+		if ( ttype == TreePatternLexer.PERCENT ) {
+			ttype = tokenizer.nextToken();
+			if ( ttype != TreePatternLexer.ID ) {
+				return null;
+			}
+			label = tokenizer.sval.toString();
+			ttype = tokenizer.nextToken();
+			if ( ttype != TreePatternLexer.COLON ) {
+				return null;
+			}
+			ttype = tokenizer.nextToken(); // move to ID following colon
+		}
+
+		// Wildcard?
+		if ( ttype == TreePatternLexer.DOT ) {
+			ttype = tokenizer.nextToken();
+			Token wildcardPayload = new CommonToken(0, ".");
+			TreeWizard.TreePattern node =
+				new TreeWizard.WildcardTreePattern(wildcardPayload);
+			if ( label!=null ) {
+				node.label = label;
+			}
+			return node;
+		}
+
+		// "ID" or "ID[arg]"
+		if ( ttype != TreePatternLexer.ID ) {
+			return null;
+		}
+		String tokenName = tokenizer.sval.toString();
+		ttype = tokenizer.nextToken();
+		if ( tokenName.equals("nil") ) {
+			return adaptor.nil();
+		}
+		String text = tokenName;
+		// check for arg
+		String arg = null;
+		if ( ttype == TreePatternLexer.ARG ) {
+			arg = tokenizer.sval.toString();
+			text = arg;
+			ttype = tokenizer.nextToken();
+		}
+		
+		// create node
+		int treeNodeType = wizard.getTokenType(tokenName);
+		if ( treeNodeType==Token.INVALID_TOKEN_TYPE ) {
+			return null;
+		}
+		Object node;
+		node = adaptor.create(treeNodeType, text);
+		if ( label!=null && node.getClass()==TreeWizard.TreePattern.class ) {
+			((TreeWizard.TreePattern)node).label = label;
+		}
+		if ( arg!=null && node.getClass()==TreeWizard.TreePattern.class ) {
+			((TreeWizard.TreePattern)node).hasTextArg = true;
+		}
+		return node;
+	}
+}
diff --git a/src/org/antlr/runtime/tree/TreeRewriter.java b/src/org/antlr/runtime/tree/TreeRewriter.java
new file mode 100755
index 0000000..91aee93
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeRewriter.java
@@ -0,0 +1,120 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.RecognizerSharedState;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.TokenStream;
+
+public class TreeRewriter extends TreeParser {
+    public interface fptr {
+        public Object rule() throws RecognitionException;
+    }
+
+    protected boolean showTransformations = false;
+
+    protected TokenStream originalTokenStream;
+    protected TreeAdaptor originalAdaptor;
+    
+    public TreeRewriter(TreeNodeStream input) {
+        this(input, new RecognizerSharedState());
+    }
+    public TreeRewriter(TreeNodeStream input, RecognizerSharedState state) {
+        super(input, state);
+        originalAdaptor = input.getTreeAdaptor();
+        originalTokenStream = input.getTokenStream();        
+    }
+
+    public Object applyOnce(Object t, fptr whichRule) {
+        if ( t==null ) return null;
+        try {
+            // share TreeParser object but not parsing-related state
+            state = new RecognizerSharedState();
+            input = new CommonTreeNodeStream(originalAdaptor, t);
+            ((CommonTreeNodeStream)input).setTokenStream(originalTokenStream);
+            setBacktrackingLevel(1);
+            TreeRuleReturnScope r = (TreeRuleReturnScope)whichRule.rule();
+            setBacktrackingLevel(0);
+            if ( failed() ) return t;
+            if ( showTransformations &&
+                 r!=null && !t.equals(r.getTree()) && r.getTree()!=null )
+            {
+                reportTransformation(t, r.getTree());
+            }
+            if ( r!=null && r.getTree()!=null ) return r.getTree();
+            else return t;
+        }
+        catch (RecognitionException e) { ; }
+        return t;
+    }
+
+    public Object applyRepeatedly(Object t, fptr whichRule) {
+        boolean treeChanged = true;
+        while ( treeChanged ) {
+            Object u = applyOnce(t, whichRule);
+            treeChanged = !t.equals(u);
+            t = u;
+        }
+        return t;
+    }
+
+    public Object downup(Object t) { return downup(t, false); }
+
+    public Object downup(Object t, boolean showTransformations) {
+        this.showTransformations = showTransformations;
+        TreeVisitor v = new TreeVisitor(new CommonTreeAdaptor());
+        TreeVisitorAction actions = new TreeVisitorAction() {
+            public Object pre(Object t)  { return applyOnce(t, topdown_fptr); }
+            public Object post(Object t) { return applyRepeatedly(t, bottomup_ftpr); }
+        };
+        t = v.visit(t, actions);
+        return t;
+    }
+
+    /** Override this if you need transformation tracing to go somewhere
+     *  other than stdout or if you're not using Tree-derived trees.
+     */
+    public void reportTransformation(Object oldTree, Object newTree) {
+        System.out.println(((Tree)oldTree).toStringTree()+" -> "+
+                           ((Tree)newTree).toStringTree());
+    }
+
+    fptr topdown_fptr = new fptr() {
+        public Object rule() throws RecognitionException { return topdown(); }
+    };
+    
+    fptr bottomup_ftpr = new fptr() {
+        public Object rule() throws RecognitionException { return bottomup(); }
+    };
+
+    // methods the downup strategy uses to do the up and down rules.
+    // to override, just define tree grammar rule topdown and turn on
+    // filter=true.
+    public Object topdown() throws RecognitionException { return null; }
+    public Object bottomup() throws RecognitionException { return null; }
+}
diff --git a/src/org/antlr/runtime/tree/TreeRuleReturnScope.java b/src/org/antlr/runtime/tree/TreeRuleReturnScope.java
new file mode 100755
index 0000000..4ea65c0
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeRuleReturnScope.java
@@ -0,0 +1,41 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.RuleReturnScope;
+
+/** This is identical to the ParserRuleReturnScope except that
+ *  the start property is a tree nodes not Token object
+ *  when you are parsing trees.  To be generic the tree node types
+ *  have to be Object.
+ */
+public class TreeRuleReturnScope extends RuleReturnScope {
+	/** First node or root node of tree matched for this rule. */
+	public Object start;
+	public Object getStart() { return start; }	
+}
diff --git a/src/org/antlr/runtime/tree/TreeVisitor.java b/src/org/antlr/runtime/tree/TreeVisitor.java
new file mode 100755
index 0000000..8c5a717
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeVisitor.java
@@ -0,0 +1,69 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime.tree;
+
+/** Do a depth first walk of a tree, applying pre() and post() actions
+ *  as we discover and finish nodes.
+ */
+public class TreeVisitor {
+    protected TreeAdaptor adaptor;
+    
+    public TreeVisitor(TreeAdaptor adaptor) {
+        this.adaptor = adaptor;
+    }
+    public TreeVisitor() { this(new CommonTreeAdaptor()); }
+    
+    /** Visit every node in tree t and trigger an action for each node
+     *  before/after having visited all of its children.
+     *  Execute both actions even if t has no children.
+     *  If a child visit yields a new child, it can update its
+     *  parent's child list or just return the new child.  The
+     *  child update code works even if the child visit alters its parent
+     *  and returns the new tree.
+     *
+     *  Return result of applying post action to this node.
+     */
+    public Object visit(Object t, TreeVisitorAction action) {
+        // System.out.println("visit "+((Tree)t).toStringTree());
+        boolean isNil = adaptor.isNil(t);
+        if ( action!=null && !isNil ) {
+            t = action.pre(t); // if rewritten, walk children of new t
+        }
+        for (int i=0; i<adaptor.getChildCount(t); i++) {
+            Object child = adaptor.getChild(t, i);
+            Object visitResult = visit(child, action);
+            Object childAfterVisit = adaptor.getChild(t, i);
+            if ( visitResult !=  childAfterVisit ) { // result & child differ?
+                adaptor.setChild(t, i, visitResult);
+            }
+        }
+        if ( action!=null && !isNil ) t = action.post(t);
+        return t;
+    }
+}
diff --git a/src/org/antlr/runtime/tree/TreeVisitorAction.java b/src/org/antlr/runtime/tree/TreeVisitorAction.java
new file mode 100755
index 0000000..ef0f93c
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeVisitorAction.java
@@ -0,0 +1,47 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.runtime.tree;
+
+/** How to execute code for node t when a visitor visits node t.  Execute
+ *  pre() before visiting children and execute post() after visiting children.
+ */
+public interface TreeVisitorAction {
+    /** Execute an action before visiting children of t.  Return t or
+     *  a rewritten t.  It is up to the visitor to decide what to do
+     *  with the return value.  Children of returned value will be
+     *  visited if using TreeVisitor.visit().
+     */
+    public Object pre(Object t);
+
+    /** Execute an action after visiting children of t.  Return t or
+     *  a rewritten t.  It is up to the visitor to decide what to do
+     *  with the return value.
+     */
+    public Object post(Object t);
+}
diff --git a/src/org/antlr/runtime/tree/TreeWizard.java b/src/org/antlr/runtime/tree/TreeWizard.java
new file mode 100755
index 0000000..666cfd6
--- /dev/null
+++ b/src/org/antlr/runtime/tree/TreeWizard.java
@@ -0,0 +1,531 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2005-2009 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.antlr.runtime.tree;
+
+import org.antlr.runtime.Token;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Build and navigate trees with this object.  Must know about the names
+ *  of tokens so you have to pass in a map or array of token names (from which
+ *  this class can build the map).  I.e., Token DECL means nothing unless the
+ *  class can translate it to a token type.
+ *
+ *  In order to create nodes and navigate, this class needs a TreeAdaptor.
+ *
+ *  This class can build a token type -> node index for repeated use or for
+ *  iterating over the various nodes with a particular type.
+ *
+ *  This class works in conjunction with the TreeAdaptor rather than moving
+ *  all this functionality into the adaptor.  An adaptor helps build and
+ *  navigate trees using methods.  This class helps you do it with string
+ *  patterns like "(A B C)".  You can create a tree from that pattern or
+ *  match subtrees against it.
+ */
+public class TreeWizard {
+	protected TreeAdaptor adaptor;
+	protected Map tokenNameToTypeMap;
+
+	public interface ContextVisitor {
+		// TODO: should this be called visit or something else?
+		public void visit(Object t, Object parent, int childIndex, Map labels);
+	}
+
+	public static abstract class Visitor implements ContextVisitor {
+		public void visit(Object t, Object parent, int childIndex, Map labels) {
+			visit(t);
+		}
+		public abstract void visit(Object t);
+	}
+
+	/** When using %label:TOKENNAME in a tree for parse(), we must
+	 *  track the label.
+	 */
+	public static class TreePattern extends CommonTree {
+		public String label;
+		public boolean hasTextArg;
+		public TreePattern(Token payload) {
+			super(payload);
+		}
+		public String toString() {
+			if ( label!=null ) {
+				return "%"+label+":"+super.toString();
+			}
+			else {
+				return super.toString();				
+			}
+		}
+	}
+
+	public static class WildcardTreePattern extends TreePattern {
+		public WildcardTreePattern(Token payload) {
+			super(payload);
+		}
+	}
+
+	/** This adaptor creates TreePattern objects for use during scan() */
+	public static class TreePatternTreeAdaptor extends CommonTreeAdaptor {
+		public Object create(Token payload) {
+			return new TreePattern(payload);
+		}
+	}
+
+	// TODO: build indexes for the wizard
+
+	/** During fillBuffer(), we can make a reverse index from a set
+	 *  of token types of interest to the list of indexes into the
+	 *  node stream.  This lets us convert a node pointer to a
+	 *  stream index semi-efficiently for a list of interesting
+	 *  nodes such as function definition nodes (you'll want to seek
+	 *  to their bodies for an interpreter).  Also useful for doing
+	 *  dynamic searches; i.e., go find me all PLUS nodes.
+	protected Map tokenTypeToStreamIndexesMap;
+
+	/** If tokenTypesToReverseIndex set to INDEX_ALL then indexing
+	 *  occurs for all token types.
+	public static final Set INDEX_ALL = new HashSet();
+
+	/** A set of token types user would like to index for faster lookup.
+	 *  If this is INDEX_ALL, then all token types are tracked.  If null,
+	 *  then none are indexed.
+	protected Set tokenTypesToReverseIndex = null;
+	*/
+
+	public TreeWizard(TreeAdaptor adaptor) {
+		this.adaptor = adaptor;
+	}
+
+	public TreeWizard(TreeAdaptor adaptor, Map tokenNameToTypeMap) {
+		this.adaptor = adaptor;
+		this.tokenNameToTypeMap = tokenNameToTypeMap;
+	}
+
+	public TreeWizard(TreeAdaptor adaptor, String[] tokenNames) {
+		this.adaptor = adaptor;
+		this.tokenNameToTypeMap = computeTokenTypes(tokenNames);
+	}
+
+	public TreeWizard(String[] tokenNames) {
+		this(new CommonTreeAdaptor(), tokenNames);
+	}
+
+	/** Compute a Map<String, Integer> that is an inverted index of
+	 *  tokenNames (which maps int token types to names).
+	 */
+	public Map computeTokenTypes(String[] tokenNames) {
+		Map m = new HashMap();
+		if ( tokenNames==null ) {
+			return m;
+		}
+		for (int ttype = Token.MIN_TOKEN_TYPE; ttype < tokenNames.length; ttype++) {
+			String name = tokenNames[ttype];
+			m.put(name, new Integer(ttype));
+		}
+		return m;
+	}
+
+	/** Using the map of token names to token types, return the type. */
+	public int getTokenType(String tokenName) {
+	 	if ( tokenNameToTypeMap==null ) {
+			 return Token.INVALID_TOKEN_TYPE;
+		 }
+		Integer ttypeI = (Integer)tokenNameToTypeMap.get(tokenName);
+		if ( ttypeI!=null ) {
+			return ttypeI.intValue();
+		}
+		return Token.INVALID_TOKEN_TYPE;
+	}
+
+	/** Walk the entire tree and make a node name to nodes mapping.
+	 *  For now, use recursion but later nonrecursive version may be
+	 *  more efficient.  Returns Map<Integer, List> where the List is
+	 *  of your AST node type.  The Integer is the token type of the node.
+	 *
+	 *  TODO: save this index so that find and visit are faster
+	 */
+	public Map index(Object t) {
+		Map m = new HashMap();
+		_index(t, m);
+		return m;
+	}
+
+	/** Do the work for index */
+	protected void _index(Object t, Map m) {
+		if ( t==null ) {
+			return;
+		}
+		int ttype = adaptor.getType(t);
+		List elements = (List)m.get(new Integer(ttype));
+		if ( elements==null ) {
+			elements = new ArrayList();
+			m.put(new Integer(ttype), elements);
+		}
+		elements.add(t);
+		int n = adaptor.getChildCount(t);
+		for (int i=0; i<n; i++) {
+			Object child = adaptor.getChild(t, i);
+			_index(child, m);
+		}
+	}
+
+	/** Return a List of tree nodes with token type ttype */
+	public List find(Object t, int ttype) {
+		final List nodes = new ArrayList();
+		visit(t, ttype, new TreeWizard.Visitor() {
+			public void visit(Object t) {
+				nodes.add(t);
+			}
+		});
+		return nodes;
+	}
+
+	/** Return a List of subtrees matching pattern. */
+	public List find(Object t, String pattern) {
+		final List subtrees = new ArrayList();
+		// Create a TreePattern from the pattern
+		TreePatternLexer tokenizer = new TreePatternLexer(pattern);
+		TreePatternParser parser =
+			new TreePatternParser(tokenizer, this, new TreePatternTreeAdaptor());
+		final TreePattern tpattern = (TreePattern)parser.pattern();
+		// don't allow invalid patterns
+		if ( tpattern==null ||
+			 tpattern.isNil() ||
+			 tpattern.getClass()==WildcardTreePattern.class )
+		{
+			return null;
+		}
+		int rootTokenType = tpattern.getType();
+		visit(t, rootTokenType, new TreeWizard.ContextVisitor() {
+			public void visit(Object t, Object parent, int childIndex, Map labels) {
+				if ( _parse(t, tpattern, null) ) {
+					subtrees.add(t);
+				}
+			}
+		});
+		return subtrees;
+	}
+
+	public Object findFirst(Object t, int ttype) {
+		return null;
+	}
+
+	public Object findFirst(Object t, String pattern) {
+		return null;
+	}
+
+	/** Visit every ttype node in t, invoking the visitor.  This is a quicker
+	 *  version of the general visit(t, pattern) method.  The labels arg
+	 *  of the visitor action method is never set (it's null) since using
+	 *  a token type rather than a pattern doesn't let us set a label.
+	 */
+	public void visit(Object t, int ttype, ContextVisitor visitor) {
+		_visit(t, null, 0, ttype, visitor);
+	}
+
+	/** Do the recursive work for visit */
+	protected void _visit(Object t, Object parent, int childIndex, int ttype, ContextVisitor visitor) {
+		if ( t==null ) {
+			return;
+		}
+		if ( adaptor.getType(t)==ttype ) {
+			visitor.visit(t, parent, childIndex, null);
+		}
+		int n = adaptor.getChildCount(t);
+		for (int i=0; i<n; i++) {
+			Object child = adaptor.getChild(t, i);
+			_visit(child, t, i, ttype, visitor);
+		}
+	}
+
+	/** For all subtrees that match the pattern, execute the visit action.
+	 *  The implementation uses the root node of the pattern in combination
+	 *  with visit(t, ttype, visitor) so nil-rooted patterns are not allowed.
+	 *  Patterns with wildcard roots are also not allowed.
+	 */
+	public void visit(Object t, final String pattern, final ContextVisitor visitor) {
+		// Create a TreePattern from the pattern
+		TreePatternLexer tokenizer = new TreePatternLexer(pattern);
+		TreePatternParser parser =
+			new TreePatternParser(tokenizer, this, new TreePatternTreeAdaptor());
+		final TreePattern tpattern = (TreePattern)parser.pattern();
+		// don't allow invalid patterns
+		if ( tpattern==null ||
+			 tpattern.isNil() ||
+			 tpattern.getClass()==WildcardTreePattern.class )
+		{
+			return;
+		}
+		final Map labels = new HashMap(); // reused for each _parse
+		int rootTokenType = tpattern.getType();
+		visit(t, rootTokenType, new TreeWizard.ContextVisitor() {
+			public void visit(Object t, Object parent, int childIndex, Map unusedlabels) {
+				// the unusedlabels arg is null as visit on token type doesn't set.
+				labels.clear();
+				if ( _parse(t, tpattern, labels) ) {
+					visitor.visit(t, parent, childIndex, labels);
+				}
+			}
+		});
+	}
+
+	/** Given a pattern like (ASSIGN %lhs:ID %rhs:.) with optional labels
+	 *  on the various nodes and '.' (dot) as the node/subtree wildcard,
+	 *  return true if the pattern matches and fill the labels Map with
+	 *  the labels pointing at the appropriate nodes.  Return false if
+	 *  the pattern is malformed or the tree does not match.
+	 *
+	 *  If a node specifies a text arg in pattern, then that must match
+	 *  for that node in t.
+	 *
+	 *  TODO: what's a better way to indicate bad pattern? Exceptions are a hassle 
+	 */
+	public boolean parse(Object t, String pattern, Map labels) {
+		TreePatternLexer tokenizer = new TreePatternLexer(pattern);
+		TreePatternParser parser =
+			new TreePatternParser(tokenizer, this, new TreePatternTreeAdaptor());
+		TreePattern tpattern = (TreePattern)parser.pattern();
+		/*
+		System.out.println("t="+((Tree)t).toStringTree());
+		System.out.println("scant="+tpattern.toStringTree());
+		*/
+		boolean matched = _parse(t, tpattern, labels);
+		return matched;
+	}
+
+	public boolean parse(Object t, String pattern) {
+		return parse(t, pattern, null);
+	}
+
+	/** Do the work for parse. Check to see if the t2 pattern fits the
+	 *  structure and token types in t1.  Check text if the pattern has
+	 *  text arguments on nodes.  Fill labels map with pointers to nodes
+	 *  in tree matched against nodes in pattern with labels.
+	 */
+	protected boolean _parse(Object t1, TreePattern tpattern, Map labels) {
+		// make sure both are non-null
+		if ( t1==null || tpattern==null ) {
+			return false;
+		}
+		// check roots (wildcard matches anything)
+		if ( tpattern.getClass() != WildcardTreePattern.class ) {
+			if ( adaptor.getType(t1) != tpattern.getType() ) return false;
+            // if pattern has text, check node text
+			if ( tpattern.hasTextArg && !adaptor.getText(t1).equals(tpattern.getText()) ) {
+				return false;
+			}
+		}
+		if ( tpattern.label!=null && labels!=null ) {
+			// map label in pattern to node in t1
+			labels.put(tpattern.label, t1);
+		}
+		// check children
+		int n1 = adaptor.getChildCount(t1);
+		int n2 = tpattern.getChildCount();
+		if ( n1 != n2 ) {
+			return false;
+		}
+		for (int i=0; i<n1; i++) {
+			Object child1 = adaptor.getChild(t1, i);
+			TreePattern child2 = (TreePattern)tpattern.getChild(i);
+			if ( !_parse(child1, child2, labels) ) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	/** Create a tree or node from the indicated tree pattern that closely
+	 *  follows ANTLR tree grammar tree element syntax:
+	 *
+	 * 		(root child1 ... child2).
+	 *
+	 *  You can also just pass in a node: ID
+	 * 
+	 *  Any node can have a text argument: ID[foo]
+	 *  (notice there are no quotes around foo--it's clear it's a string).
+	 *
+	 *  nil is a special name meaning "give me a nil node".  Useful for
+	 *  making lists: (nil A B C) is a list of A B C.
+ 	 */
+	public Object create(String pattern) {
+		TreePatternLexer tokenizer = new TreePatternLexer(pattern);
+		TreePatternParser parser = new TreePatternParser(tokenizer, this, adaptor);
+		Object t = parser.pattern();
+		return t;
+	}
+
+	/** Compare t1 and t2; return true if token types/text, structure match exactly.
+	 *  The trees are examined in their entirety so that (A B) does not match
+	 *  (A B C) nor (A (B C)). 
+	 // TODO: allow them to pass in a comparator
+	 *  TODO: have a version that is nonstatic so it can use instance adaptor
+	 *
+	 *  I cannot rely on the tree node's equals() implementation as I make
+	 *  no constraints at all on the node types nor interface etc... 
+	 */
+	public static boolean equals(Object t1, Object t2, TreeAdaptor adaptor) {
+		return _equals(t1, t2, adaptor);
+	}
+
+	/** Compare type, structure, and text of two trees, assuming adaptor in
+	 *  this instance of a TreeWizard.
+	 */
+	public boolean equals(Object t1, Object t2) {
+		return _equals(t1, t2, adaptor);
+	}
+
+	protected static boolean _equals(Object t1, Object t2, TreeAdaptor adaptor) {
+		// make sure both are non-null
+		if ( t1==null || t2==null ) {
+			return false;
+		}
+		// check roots
+		if ( adaptor.getType(t1) != adaptor.getType(t2) ) {
+			return false;
+		}
+		if ( !adaptor.getText(t1).equals(adaptor.getText(t2)) ) {
+			return false;
+		}
+		// check children
+		int n1 = adaptor.getChildCount(t1);
+		int n2 = adaptor.getChildCount(t2);
+		if ( n1 != n2 ) {
+			return false;
+		}
+		for (int i=0; i<n1; i++) {
+			Object child1 = adaptor.getChild(t1, i);
+			Object child2 = adaptor.getChild(t2, i);
+			if ( !_equals(child1, child2, adaptor) ) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	// TODO: next stuff taken from CommonTreeNodeStream
+	
+		/** Given a node, add this to the reverse index tokenTypeToStreamIndexesMap.
+	 *  You can override this method to alter how indexing occurs.  The
+	 *  default is to create a
+	 *
+	 *    Map<Integer token type,ArrayList<Integer stream index>>
+	 *
+	 *  This data structure allows you to find all nodes with type INT in order.
+	 *
+	 *  If you really need to find a node of type, say, FUNC quickly then perhaps
+	 *
+	 *    Map<Integertoken type,Map<Object tree node,Integer stream index>>
+	 *
+	 *  would be better for you.  The interior maps map a tree node to
+	 *  the index so you don't have to search linearly for a specific node.
+	 *
+	 *  If you change this method, you will likely need to change
+	 *  getNodeIndex(), which extracts information.
+	protected void fillReverseIndex(Object node, int streamIndex) {
+		//System.out.println("revIndex "+node+"@"+streamIndex);
+		if ( tokenTypesToReverseIndex==null ) {
+			return; // no indexing if this is empty (nothing of interest)
+		}
+		if ( tokenTypeToStreamIndexesMap==null ) {
+			tokenTypeToStreamIndexesMap = new HashMap(); // first indexing op
+		}
+		int tokenType = adaptor.getType(node);
+		Integer tokenTypeI = new Integer(tokenType);
+		if ( !(tokenTypesToReverseIndex==INDEX_ALL ||
+			   tokenTypesToReverseIndex.contains(tokenTypeI)) )
+		{
+			return; // tokenType not of interest
+		}
+		Integer streamIndexI = new Integer(streamIndex);
+		ArrayList indexes = (ArrayList)tokenTypeToStreamIndexesMap.get(tokenTypeI);
+		if ( indexes==null ) {
+			indexes = new ArrayList(); // no list yet for this token type
+			indexes.add(streamIndexI); // not there yet, add
+			tokenTypeToStreamIndexesMap.put(tokenTypeI, indexes);
+		}
+		else {
+			if ( !indexes.contains(streamIndexI) ) {
+				indexes.add(streamIndexI); // not there yet, add
+			}
+		}
+	}
+
+	/** Track the indicated token type in the reverse index.  Call this
+	 *  repeatedly for each type or use variant with Set argument to
+	 *  set all at once.
+	 * @param tokenType
+	public void reverseIndex(int tokenType) {
+		if ( tokenTypesToReverseIndex==null ) {
+			tokenTypesToReverseIndex = new HashSet();
+		}
+		else if ( tokenTypesToReverseIndex==INDEX_ALL ) {
+			return;
+		}
+		tokenTypesToReverseIndex.add(new Integer(tokenType));
+	}
+
+	/** Track the indicated token types in the reverse index. Set
+	 *  to INDEX_ALL to track all token types.
+	public void reverseIndex(Set tokenTypes) {
+		tokenTypesToReverseIndex = tokenTypes;
+	}
+
+	/** Given a node pointer, return its index into the node stream.
+	 *  This is not its Token stream index.  If there is no reverse map
+	 *  from node to stream index or the map does not contain entries
+	 *  for node's token type, a linear search of entire stream is used.
+	 *
+	 *  Return -1 if exact node pointer not in stream.
+	public int getNodeIndex(Object node) {
+		//System.out.println("get "+node);
+		if ( tokenTypeToStreamIndexesMap==null ) {
+			return getNodeIndexLinearly(node);
+		}
+		int tokenType = adaptor.getType(node);
+		Integer tokenTypeI = new Integer(tokenType);
+		ArrayList indexes = (ArrayList)tokenTypeToStreamIndexesMap.get(tokenTypeI);
+		if ( indexes==null ) {
+			//System.out.println("found linearly; stream index = "+getNodeIndexLinearly(node));
+			return getNodeIndexLinearly(node);
+		}
+		for (int i = 0; i < indexes.size(); i++) {
+			Integer streamIndexI = (Integer)indexes.get(i);
+			Object n = get(streamIndexI.intValue());
+			if ( n==node ) {
+				//System.out.println("found in index; stream index = "+streamIndexI);
+				return streamIndexI.intValue(); // found it!
+			}
+		}
+		return -1;
+	}
+
+	*/
+}