Add lexer implementation based on existing smali lexer

8 files changed, 633 insertions, 0 deletions

diff --git a/smali/src/main/jflex/smaliLexer.flex b/smali/src/main/jflex/smaliLexer.flex
index df571e66..8b7c66e3 100644
--- a/smali/src/main/jflex/smaliLexer.flex
+++ b/smali/src/main/jflex/smaliLexer.flex
@@ -172,6 +172,20 @@ import static org.jf.smali.smaliParser.*;
     public String getErrorHeader(InvalidToken token) {
         return getSourceName()+"["+ token.getLine()+","+token.getCharPositionInLine()+"]";
     }
+
+    public void reset(CharSequence charSequence, int start, int end, int initialState) {
+        zzReader = BlankReader.INSTANCE;
+        zzBuffer = new char[charSequence.length()];
+        for (int i=0; i<charSequence.length(); i++) {
+            zzBuffer[i] = charSequence.charAt(i);
+        }
+
+        yychar = zzCurrentPos = zzMarkedPos = zzStartRead = start;
+        zzEndRead = end;
+        zzAtBOL = true;
+        zzAtEOF = false;
+        yybegin(initialState);
+    }
 %}
 
 HexPrefix = 0 [xX]
diff --git a/smalidea/META-INF/plugin.xml b/smalidea/META-INF/plugin.xml
index 8b142a37..788f39d9 100644
--- a/smalidea/META-INF/plugin.xml
+++ b/smalidea/META-INF/plugin.xml
@@ -16,6 +16,7 @@
 
   <extensions defaultExtensionNs="com.intellij">
       <fileTypeFactory implementation="org.jf.smalidea.SmaliFileTypeFactory"/>
+      <syntaxHighlighter key="smali" implementationClass="org.jf.smalidea.SmaliHighlighter"/>
   </extensions>
 
   <application-components>
diff --git a/smalidea/smalidea.iml b/smalidea/smalidea.iml
index 3411cee0..6f133f4e 100644
--- a/smalidea/smalidea.iml
+++ b/smalidea/smalidea.iml
@@ -10,6 +10,7 @@
     </content>
     <orderEntry type="jdk" jdkName="IDEA Plugin jdk" jdkType="IDEA JDK" />
     <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module" module-name="smali-smali" />
   </component>
 </module>
 
diff --git a/smalidea/src/main/java/org/jf/smalidea/SmaliHighlighter.java b/smalidea/src/main/java/org/jf/smalidea/SmaliHighlighter.java
new file mode 100644
index 00000000..0683a714
--- /dev/null
+++ b/smalidea/src/main/java/org/jf/smalidea/SmaliHighlighter.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.jf.smalidea;
+
+import com.intellij.lexer.Lexer;
+import com.intellij.openapi.editor.colors.TextAttributesKey;
+import com.intellij.openapi.fileTypes.SyntaxHighlighterBase;
+import com.intellij.psi.tree.IElementType;
+import org.jetbrains.annotations.NotNull;
+
+public class SmaliHighlighter extends SyntaxHighlighterBase {
+    @NotNull
+    public Lexer getHighlightingLexer() {
+        return new SmaliLexer();
+    }
+
+    @NotNull
+    public TextAttributesKey[] getTokenHighlights(IElementType tokenType) {
+        return new TextAttributesKey[] {};
+    }
+}
diff --git a/smalidea/src/main/java/org/jf/smalidea/SmaliLexer.java b/smalidea/src/main/java/org/jf/smalidea/SmaliLexer.java
new file mode 100644
index 00000000..7f8904d3
--- /dev/null
+++ b/smalidea/src/main/java/org/jf/smalidea/SmaliLexer.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.jf.smalidea;
+
+import com.intellij.lexer.LexerBase;
+import com.intellij.psi.TokenType;
+import com.intellij.psi.tree.IElementType;
+import org.antlr.runtime.CommonToken;
+import org.jetbrains.annotations.NotNull;
+import org.jf.smali.smaliFlexLexer;
+import org.jf.smali.smaliParser;
+import org.jf.util.BlankReader;
+
+public class SmaliLexer extends LexerBase {
+    private final smaliFlexLexer lexer = new smaliFlexLexer(BlankReader.INSTANCE);
+    private CommonToken token = null;
+    private int state = 0;
+    private int endOffset;
+    private CharSequence text;
+
+    public SmaliLexer() {
+        super();
+        lexer.setSuppressErrors(true);
+    }
+
+    @Override public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) {
+        text = buffer;
+        lexer.reset(buffer, startOffset, endOffset, initialState);
+        this.endOffset = endOffset;
+        this.token = null;
+        this.state = 0;
+    }
+
+    @NotNull @Override public CharSequence getTokenSequence() {
+        return getTokenText();
+    }
+
+    @NotNull @Override public String getTokenText() {
+        ensureToken();
+        return token.getText();
+    }
+
+    @Override
+    public int getState() {
+        ensureToken();
+        return state;
+    }
+
+    @Override
+    public IElementType getTokenType() {
+        ensureToken();
+        return mapTokenTypeToElementType(token.getType());
+    }
+
+    private IElementType mapTokenTypeToElementType(int tokenType) {
+        if (tokenType == smaliParser.WHITE_SPACE) {
+            return TokenType.WHITE_SPACE;
+        }
+        if (tokenType == smaliParser.INVALID_TOKEN) {
+            return TokenType.BAD_CHARACTER;
+        }
+        if (tokenType == smaliParser.EOF) {
+            return null;
+        }
+        return SmaliTokens.getElementType(tokenType);
+    }
+
+    @Override
+    public int getTokenStart() {
+        ensureToken();
+        return token.getStartIndex();
+    }
+
+    @Override
+    public int getTokenEnd() {
+        ensureToken();
+        return token.getStopIndex()+1;
+    }
+
+    @Override
+    public void advance() {
+        token = null;
+        state = 0;
+    }
+
+    @NotNull @Override public CharSequence getBufferSequence() {
+        return text;
+    }
+
+    @Override
+    public int getBufferEnd() {
+        return endOffset;
+    }
+
+    private void ensureToken() {
+        if (token == null) {
+            token = (CommonToken)lexer.nextToken();
+            state = lexer.yystate();
+        }
+    }
+}
diff --git a/smalidea/src/main/java/org/jf/smalidea/SmaliTokens.java b/smalidea/src/main/java/org/jf/smalidea/SmaliTokens.java
new file mode 100644
index 00000000..b4d3f33f
--- /dev/null
+++ b/smalidea/src/main/java/org/jf/smalidea/SmaliTokens.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.jf.smalidea;
+
+import com.intellij.psi.tree.IElementType;
+import org.jf.smali.smaliParser;
+
+import java.lang.reflect.Field;
+
+public class SmaliTokens {
+    private static final IElementType[] ELEMENT_TYPES;
+
+    public static IElementType getElementType(int tokenType) {
+        return ELEMENT_TYPES[tokenType];
+    }
+
+    public static IElementType ACCESS_SPEC;
+    public static IElementType ANNOTATION_DIRECTIVE;
+    public static IElementType ANNOTATION_VISIBILITY;
+    public static IElementType ARRAY_DATA_DIRECTIVE;
+    public static IElementType ARRAY_DESCRIPTOR;
+    public static IElementType ARROW;
+    public static IElementType BOOL_LITERAL;
+    public static IElementType BYTE_LITERAL;
+    public static IElementType CATCH_DIRECTIVE;
+    public static IElementType CATCHALL_DIRECTIVE;
+    public static IElementType CHAR_LITERAL;
+    public static IElementType CLASS_DESCRIPTOR;
+    public static IElementType CLASS_DIRECTIVE;
+    public static IElementType CLOSE_BRACE;
+    public static IElementType CLOSE_PAREN;
+    public static IElementType COLON;
+    public static IElementType COMMA;
+    public static IElementType DOTDOT;
+    public static IElementType DOUBLE_LITERAL;
+    public static IElementType DOUBLE_LITERAL_OR_ID;
+    public static IElementType END_ANNOTATION_DIRECTIVE;
+    public static IElementType END_ARRAY_DATA_DIRECTIVE;
+    public static IElementType END_FIELD_DIRECTIVE;
+    public static IElementType END_LOCAL_DIRECTIVE;
+    public static IElementType END_METHOD_DIRECTIVE;
+    public static IElementType END_PACKED_SWITCH_DIRECTIVE;
+    public static IElementType END_PARAMETER_DIRECTIVE;
+    public static IElementType END_SPARSE_SWITCH_DIRECTIVE;
+    public static IElementType END_SUBANNOTATION_DIRECTIVE;
+    public static IElementType ENUM_DIRECTIVE;
+    public static IElementType EPILOGUE_DIRECTIVE;
+    public static IElementType EQUAL;
+    public static IElementType FIELD_DIRECTIVE;
+    public static IElementType FIELD_OFFSET;
+    public static IElementType FLOAT_LITERAL;
+    public static IElementType FLOAT_LITERAL_OR_ID;
+    public static IElementType IMPLEMENTS_DIRECTIVE;
+    public static IElementType INLINE_INDEX;
+    public static IElementType INSTRUCTION_FORMAT10t;
+    public static IElementType INSTRUCTION_FORMAT10x;
+    public static IElementType INSTRUCTION_FORMAT10x_ODEX;
+    public static IElementType INSTRUCTION_FORMAT11n;
+    public static IElementType INSTRUCTION_FORMAT11x;
+    public static IElementType INSTRUCTION_FORMAT12x;
+    public static IElementType INSTRUCTION_FORMAT12x_OR_ID;
+    public static IElementType INSTRUCTION_FORMAT20bc;
+    public static IElementType INSTRUCTION_FORMAT20t;
+    public static IElementType INSTRUCTION_FORMAT21c_FIELD;
+    public static IElementType INSTRUCTION_FORMAT21c_FIELD_ODEX;
+    public static IElementType INSTRUCTION_FORMAT21c_STRING;
+    public static IElementType INSTRUCTION_FORMAT21c_TYPE;
+    public static IElementType INSTRUCTION_FORMAT21ih;
+    public static IElementType INSTRUCTION_FORMAT21lh;
+    public static IElementType INSTRUCTION_FORMAT21s;
+    public static IElementType INSTRUCTION_FORMAT21t;
+    public static IElementType INSTRUCTION_FORMAT22b;
+    public static IElementType INSTRUCTION_FORMAT22c_FIELD;
+    public static IElementType INSTRUCTION_FORMAT22c_FIELD_ODEX;
+    public static IElementType INSTRUCTION_FORMAT22c_TYPE;
+    public static IElementType INSTRUCTION_FORMAT22cs_FIELD;
+    public static IElementType INSTRUCTION_FORMAT22s;
+    public static IElementType INSTRUCTION_FORMAT22s_OR_ID;
+    public static IElementType INSTRUCTION_FORMAT22t;
+    public static IElementType INSTRUCTION_FORMAT22x;
+    public static IElementType INSTRUCTION_FORMAT23x;
+    public static IElementType INSTRUCTION_FORMAT30t;
+    public static IElementType INSTRUCTION_FORMAT31c;
+    public static IElementType INSTRUCTION_FORMAT31i;
+    public static IElementType INSTRUCTION_FORMAT31i_OR_ID;
+    public static IElementType INSTRUCTION_FORMAT31t;
+    public static IElementType INSTRUCTION_FORMAT32x;
+    public static IElementType INSTRUCTION_FORMAT35c_METHOD;
+    public static IElementType INSTRUCTION_FORMAT35c_METHOD_ODEX;
+    public static IElementType INSTRUCTION_FORMAT35c_TYPE;
+    public static IElementType INSTRUCTION_FORMAT35mi_METHOD;
+    public static IElementType INSTRUCTION_FORMAT35ms_METHOD;
+    public static IElementType INSTRUCTION_FORMAT3rc_METHOD;
+    public static IElementType INSTRUCTION_FORMAT3rc_METHOD_ODEX;
+    public static IElementType INSTRUCTION_FORMAT3rc_TYPE;
+    public static IElementType INSTRUCTION_FORMAT3rmi_METHOD;
+    public static IElementType INSTRUCTION_FORMAT3rms_METHOD;
+    public static IElementType INSTRUCTION_FORMAT51l;
+    public static IElementType LINE_COMMENT;
+    public static IElementType LINE_DIRECTIVE;
+    public static IElementType LOCAL_DIRECTIVE;
+    public static IElementType LOCALS_DIRECTIVE;
+    public static IElementType LONG_LITERAL;
+    public static IElementType METHOD_DIRECTIVE;
+    public static IElementType MEMBER_NAME;
+    public static IElementType NEGATIVE_INTEGER_LITERAL;
+    public static IElementType NULL_LITERAL;
+    public static IElementType OPEN_BRACE;
+    public static IElementType OPEN_PAREN;
+    public static IElementType PACKED_SWITCH_DIRECTIVE;
+    public static IElementType PARAM_LIST_END;
+    public static IElementType PARAM_LIST_START;
+    public static IElementType PARAM_LIST_OR_ID_END;
+    public static IElementType PARAM_LIST_OR_ID_START;
+    public static IElementType PARAMETER_DIRECTIVE;
+    public static IElementType POSITIVE_INTEGER_LITERAL;
+    public static IElementType PRIMITIVE_TYPE;
+    public static IElementType PROLOGUE_DIRECTIVE;
+    public static IElementType REGISTER;
+    public static IElementType REGISTERS_DIRECTIVE;
+    public static IElementType RESTART_LOCAL_DIRECTIVE;
+    public static IElementType SHORT_LITERAL;
+    public static IElementType SIMPLE_NAME;
+    public static IElementType SOURCE_DIRECTIVE;
+    public static IElementType SPARSE_SWITCH_DIRECTIVE;
+    public static IElementType STRING_LITERAL;
+    public static IElementType SUBANNOTATION_DIRECTIVE;
+    public static IElementType SUPER_DIRECTIVE;
+    public static IElementType VERIFICATION_ERROR_TYPE;
+    public static IElementType VOID_TYPE;
+    public static IElementType VTABLE_INDEX;
+
+    static {
+        int tokenCount = smaliParser.tokenNames.length;
+        ELEMENT_TYPES = new IElementType[tokenCount];
+
+        for (int tokenId=0; tokenId<tokenCount; tokenId++) {
+            String tokenName = smaliParser.tokenNames[tokenId];
+            Field field;
+
+            try {
+                field = SmaliTokens.class.getField(tokenName);
+            } catch (NoSuchFieldException ex) {
+                continue;
+            }
+
+            IElementType elementType = new IElementType(tokenName, SmaliLanguage.INSTANCE);
+            ELEMENT_TYPES[tokenId] = elementType;
+
+            try {
+                field.set(null, elementType);
+            } catch (IllegalAccessException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+    }
+}
diff --git a/smalidea/src/test/java/org/jf/smalidea/SmaliLexerTest.java b/smalidea/src/test/java/org/jf/smalidea/SmaliLexerTest.java
new file mode 100644
index 00000000..14e424ef
--- /dev/null
+++ b/smalidea/src/test/java/org/jf/smalidea/SmaliLexerTest.java
@@ -0,0 +1,203 @@
+/*
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.jf.smalidea;
+
+import com.intellij.lexer.Lexer;
+import com.intellij.testFramework.LexerTestCase;
+
+import java.util.Random;
+
+/**
+ * This is mostly just a smoke test to make sure the lexer is working. The lexer itself has its
+ * own tests in the smali module
+ */
+public class SmaliLexerTest extends LexerTestCase {
+    public void testHelloWorld() {
+        String text =
+                ".class public LHelloWorld;\n" +
+                ".super Ljava/lang/Object;\n" +
+                ".method public static main([Ljava/lang/String;)V\n" +
+                "    .registers 2\n" +
+                "    sget-object v0, Ljava/lang/System;->out:Ljava/io/PrintStream;\n" +
+                "    const-string v1, \"Hello World!\"\n" +
+                "    invoke-virtual {v0, v1}, Ljava/io/PrintStream;->println(Ljava/lang/String;)V\n" +
+                "    return-void\n" +
+                ".end method";
+
+        doTest(text,
+                "CLASS_DIRECTIVE ('.class')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "ACCESS_SPEC ('public')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "CLASS_DESCRIPTOR ('LHelloWorld;')\n" +
+                "WHITE_SPACE ('\\n')\n" +
+                "SUPER_DIRECTIVE ('.super')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "CLASS_DESCRIPTOR ('Ljava/lang/Object;')\n" +
+                "WHITE_SPACE ('\\n')\n" +
+                "METHOD_DIRECTIVE ('.method')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "ACCESS_SPEC ('public')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "ACCESS_SPEC ('static')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "SIMPLE_NAME ('main')\n" +
+                "OPEN_PAREN ('(')\n" +
+                "ARRAY_DESCRIPTOR ('[Ljava/lang/String;')\n" +
+                "CLOSE_PAREN (')')\n" +
+                "VOID_TYPE ('V')\n" +
+                "WHITE_SPACE ('\\n    ')\n" +
+                "REGISTERS_DIRECTIVE ('.registers')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "POSITIVE_INTEGER_LITERAL ('2')\n" +
+                "WHITE_SPACE ('\\n    ')\n" +
+                "INSTRUCTION_FORMAT21c_FIELD ('sget-object')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "REGISTER ('v0')\n" +
+                "COMMA (',')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "CLASS_DESCRIPTOR ('Ljava/lang/System;')\n" +
+                "ARROW ('->')\n" +
+                "SIMPLE_NAME ('out')\n" +
+                "COLON (':')\n" +
+                "CLASS_DESCRIPTOR ('Ljava/io/PrintStream;')\n" +
+                "WHITE_SPACE ('\\n    ')\n" +
+                "INSTRUCTION_FORMAT21c_STRING ('const-string')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "REGISTER ('v1')\n" +
+                "COMMA (',')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "STRING_LITERAL ('\"Hello World!\"')\n" +
+                "WHITE_SPACE ('\\n    ')\n" +
+                "INSTRUCTION_FORMAT35c_METHOD ('invoke-virtual')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "OPEN_BRACE ('{')\n" +
+                "REGISTER ('v0')\n" +
+                "COMMA (',')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "REGISTER ('v1')\n" +
+                "CLOSE_BRACE ('}')\n" +
+                "COMMA (',')\n" +
+                "WHITE_SPACE (' ')\n" +
+                "CLASS_DESCRIPTOR ('Ljava/io/PrintStream;')\n" +
+                "ARROW ('->')\n" +
+                "SIMPLE_NAME ('println')\n" +
+                "OPEN_PAREN ('(')\n" +
+                "CLASS_DESCRIPTOR ('Ljava/lang/String;')\n" +
+                "CLOSE_PAREN (')')\n" +
+                "VOID_TYPE ('V')\n" +
+                "WHITE_SPACE ('\\n    ')\n" +
+                "INSTRUCTION_FORMAT10x ('return-void')\n" +
+                "WHITE_SPACE ('\\n')\n" +
+                "END_METHOD_DIRECTIVE ('.end method')"
+        );
+    }
+
+    @Override protected Lexer createLexer() {
+        return new SmaliLexer();
+    }
+
+    @Override protected String getDirPath() {
+        return "";
+    }
+
+    public void testErrorToken() {
+        String text = ".class public .blah";
+        doTest(text,
+                "CLASS_DIRECTIVE ('.class')\n" +
+                        "WHITE_SPACE (' ')\n" +
+                        "ACCESS_SPEC ('public')\n" +
+                        "WHITE_SPACE (' ')\n" +
+                        "BAD_CHARACTER ('.blah')\n");
+    }
+
+    /**
+     * Type out an example smali file character by character, ensuring that no exceptions are thrown
+     */
+    public void testPartialText() {
+        String text =
+                ".class public LHelloWorld;\n" +
+                        ".super Ljava/lang/Object;\n" +
+                        ".method public static main([Ljava/lang/String;)V\n" +
+                        "    .registers 2\n" +
+                        "    sget-object v0, Ljava/lang/System;->out:Ljava/io/PrintStream;\n" +
+                        "    const-string v1, \"Hello World!\"\n" +
+                        "    invoke-virtual {v0, v1}, Ljava/io/PrintStream;->println(Ljava/lang/String;)V\n" +
+                        "    return-void\n" +
+                        ".end method";
+
+        for (int i=1; i<text.length(); i++) {
+            printTokens(text.substring(i), 0);
+        }
+    }
+
+    /**
+     * Generate some random text and make sure the lexer doesn't throw any exceptions
+     */
+    public void testRandomText() {
+        for (int i=0; i<100; i++) {
+            String randomString = randomString(1000);
+
+            printTokens(randomString, 0);
+        }
+    }
+
+    private Random random = new Random(123456789);
+    private String randomString(int length) {
+        StringBuilder sb = new StringBuilder();
+        for (int i=0; i<length; i++) {
+            int type = random.nextInt(10);
+
+            if (type == 9) {
+                int randomCodepoint;
+                do {
+                    randomCodepoint = random.nextInt();
+                } while(!Character.isValidCodePoint(randomCodepoint));
+                sb.appendCodePoint(randomCodepoint);
+            } else if (type == 8) {
+                char randomChar;
+                do {
+                    randomChar = (char)random.nextInt(2^16);
+                } while(!Character.isValidCodePoint(randomChar));
+                sb.append(randomChar);
+            } else if (type > 4) {
+                sb.append((char)random.nextInt(256));
+            } else if (type == 4) {
+                sb.append(' ');
+            } else {
+                sb.append((char)random.nextInt(128));
+            }
+        }
+
+        return sb.toString();
+    }
+}
diff --git a/util/src/main/java/org/jf/util/BlankReader.java b/util/src/main/java/org/jf/util/BlankReader.java
new file mode 100644
index 00000000..ca55dd0f
--- /dev/null
+++ b/util/src/main/java/org/jf/util/BlankReader.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.jf.util;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.io.Reader;
+
+public class BlankReader extends Reader {
+    public static final BlankReader INSTANCE = new BlankReader();
+
+    @Override public int read(@Nonnull char[] chars, int i, int i2) throws IOException {
+        return -1;
+    }
+
+    @Override
+    public void close() throws IOException {
+    }
+}