[engine][VELOCITY-917] Compile-time configurable parser : working StandardParser

git-svn-id: https://svn.apache.org/repos/asf/velocity/engine/branches/parser_experiments@1860722 13f79535-47bb-0310-9956-ffa450edef68

8 files changed, 145 insertions, 201 deletions

diff --git a/velocity-engine-core/pom.xml b/velocity-engine-core/pom.xml
index a37b3df4..416b0097 100644
--- a/velocity-engine-core/pom.xml
+++ b/velocity-engine-core/pom.xml
@@ -31,13 +31,12 @@
     <name>Apache Velocity - Engine</name>
 
     <properties>
-        <!-- command line switch -Dparser.create=true generates the parser files -->
-        <parser.create>false</parser.create>
-
-        <!-- command line switch -Dparser.nodefiles=true generates AST Node classes (new structures added to parser) -->
-        <parser.nodefiles>false</parser.nodefiles>
-
         <parser.debug>false</parser.debug>
+        <parser.name>Standard</parser.name>
+        <parser.char.asterisk>*</parser.char.asterisk>
+        <parser.char.at>@</parser.char.at>
+        <parser.char.dollar>$</parser.char.dollar>
+        <parser.char.hash>#</parser.char.hash>
         
         <!-- You can modify those properties locally to test
              the DataSourceResourceLoader against other engines.
@@ -55,7 +54,15 @@
 
     <build>
         <plugins>
-            <!-- resources -->
+            <!-- maven lifecycle debugging plugin, see 'mvn buildplan:help' -->
+
+            <plugin>
+                <groupId>fr.jcgay.maven.plugins</groupId>
+                <artifactId>buildplan-maven-plugin</artifactId>
+                <version>1.3</version>
+            </plugin>
+
+            <!-- prepare parser grammar file -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-resources-plugin</artifactId>
@@ -66,6 +73,24 @@
                         <delimiter>${*}</delimiter>
                     </delimiters>
                 </configuration>
+                <executions>
+                    <execution>
+                        <id>gen-parser-grammar</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <resources>
+                                <resource>
+                                    <directory>src/main/parser</directory>
+                                    <filtering>true</filtering>
+                                </resource>
+                            </resources>
+                            <outputDirectory>${project.build.directory}/parser</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
             </plugin>
 
             <!-- shading of commons-io -->
@@ -106,6 +131,20 @@
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>javacc-maven-plugin</artifactId>
                 <version>2.6</version>
+                <configuration>
+                    <isStatic>false</isStatic>
+                    <buildParser>true</buildParser>
+                    <buildNodeFiles>false</buildNodeFiles>
+                    <multi>true</multi>
+                    <debugParser>${parser.debug}</debugParser>
+                    <debugLookAhead>${parser.debug}</debugLookAhead>
+                    <debugTokenManager>${parser.debug}</debugTokenManager>
+                    <jdkVersion>${maven.compiler.target}</jdkVersion>
+                    <nodeUsesParser>true</nodeUsesParser>
+                    <nodePackage>org.apache.velocity.runtime.parser.node</nodePackage>
+                    <sourceDirectory>${project.build.directory}/parser</sourceDirectory>
+                    <tokenManagerUsesParser>true</tokenManagerUsesParser>
+                </configuration>
                 <executions>
                     <!-- build the standard parser -->
                     <execution>
@@ -115,46 +154,17 @@
                         </goals>
                         <configuration>
                             <includes>
-                                <include>StandardParser.jjt</include>
-                            </includes>
-                        </configuration>
-                    </execution>
-                    <!-- build a tailored parser -->
-                    <execution>
-                        <id>jjtree-javacc-custom</id>
-                        <goals>
-                            <goal>jjtree-javacc</goal>
-                        </goals>
-                        <configuration>
-                            <includes>
-                                <include>TailoredParser.jjt</include>
+                                <include>Parser.jjt</include>
                             </includes>
-                            <interimDirectory>${project.build.directory}/tailored-parser/jjtree</interimDirectory>
-                            <outputDirectory>${project.build.directory}/tailored-parser/javacc</outputDirectory>
                         </configuration>
                     </execution>
                 </executions>
-                <configuration>
-                    <isStatic>false</isStatic>
-                    <buildParser>true</buildParser>
-                    <buildNodeFiles>${parser.nodefiles}</buildNodeFiles>
-                    <multi>true</multi>
-                    <debugParser>${parser.debug}</debugParser>
-                    <debugLookAhead>${parser.debug}</debugLookAhead>
-                    <debugTokenManager>${parser.debug}</debugTokenManager>
-                    <jdkVersion>${maven.compiler.target}</jdkVersion>
-                    <nodeUsesParser>true</nodeUsesParser>
-                    <nodePackage>org.apache.velocity.runtime.parser.node</nodePackage>
-                    <sourceDirectory>${basedir}/src/main/parser</sourceDirectory>
-                    <tokenManagerUsesParser>true</tokenManagerUsesParser>
-                </configuration>
             </plugin>
 
             <!--
-                patch generated parser and token managers files, so that
-                + both token managers implement the ParserTokenManager interface
+                patch generated parser and token manager files, so that
+                + token manager implement the ParserTokenManager interface
                 + StandardParser uses the ParserTokenManager interface
-            -->
             <plugin>
                 <groupId>com.google.code.maven-replacer-plugin</groupId>
                 <artifactId>replacer</artifactId>
@@ -162,7 +172,7 @@
                 <executions>
                     <execution>
                         <id>patch-parser</id>
-                        <phase>process-sources</phase>
+                        <phase>generate-sources</phase>
                         <goals>
                             <goal>replace</goal> 
                         </goals>
@@ -178,14 +188,13 @@
                     </execution>
                     <execution>
                         <id>patch-token-manager</id>
-                        <phase>process-sources</phase>
+                        <phase>generate-sources</phase>
                         <goals>
                             <goal>replace</goal> 
                         </goals> 
                         <configuration>
                             <includes>
                                 <include>${project.build.directory}/generated-sources/javacc/org/apache/velocity/runtime/parser/StandardParserTokenManager.java</include>
-                                <include>${project.build.directory}/tailored-parser/javacc/org/apache/velocity/runtime/parser/TailoredParserTokenManager.java</include>
                             </includes>
                             <replacements>
                                 <replacement>
@@ -197,54 +206,7 @@
                     </execution>
                 </executions>
             </plugin>
-
-            <!-- pre-compile both parser token manager classes -->
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>compile-patch-sources</id>
-                        <phase>process-sources</phase>
-                        <goals>
-                            <goal>compile</goal>
-                        </goals>
-                        <configuration>
-                            <buildDirectory>${project.build.directory}/tailored-parser/classes</buildDirectory>
-                            <includes>
-                                <include>**/*TokenManager*.java/</include>
-                            </includes>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <!-- remove duplicate javacc files -->
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-clean-plugin</artifactId>
-                <version>3.1.0</version>
-                <executions>
-                    <execution>
-                        <id>clean-extra-javacc</id>
-                        <phase>process-sources</phase>
-                        <goals>
-                            <goal>clean</goal>
-                        </goals>
-                        <configuration>
-                            <excludeDefaultDirectories>true</excludeDefaultDirectories>
-                            <filesets>
-                                <fileset>
-                                    <directory>${project.build.directory}/tailored-parser/javacc</directory>
-                                    <includes>
-                                        <include>org/apache/velocity/runtime/parser/Token*.java</include>
-                                    </includes>
-                                </fileset>
-                            </filesets>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
+            -->
 
             <!-- handle OSGi informations -->
             <plugin>
@@ -263,16 +225,6 @@
                 </configuration>
             </plugin>
 
-            <!-- exclude tailored parser from final archive -->
-            <plugin>
-                <artifactId>maven-jar-plugin</artifactId>
-                <configuration>
-                    <excludes>
-                        <exclude>**/*TailoredParser*.class</exclude>
-                    </excludes>
-                </configuration>
-            </plugin>
-
             <!-- tests -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
diff --git a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/RuntimeConstants.java b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/RuntimeConstants.java
index b68631b9..0ac45111 100644
--- a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/RuntimeConstants.java
+++ b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/RuntimeConstants.java
@@ -372,6 +372,18 @@ public interface RuntimeConstants extends DeprecatedRuntimeConstants
      */
 
     /**
+     * Property specifying the parser class to use
+     * @since 2.2
+     */
+    String PARSER_CLASS = "parser.class";
+
+    /**
+     * Default parser class
+     * @since 2.2
+     */
+    String DEFAULT_PARSER_CLASS = "org.apache.velocity.runtime.parser.StandardParser";
+
+    /**
      * The <code>parser.pool.class</code> property specifies the name of the {@link org.apache.velocity.util.SimplePool}
      * implementation to use.
      */
@@ -388,30 +400,6 @@ public interface RuntimeConstants extends DeprecatedRuntimeConstants
      */
     String PARSER_HYPHEN_ALLOWED = "parser.allow_hyphen_in_identifiers";
 
-    /**
-     * Set the character (onlysingle byte UTF-8 supported at present) to use instead of '$' for references.
-     * @since 2.2
-     */
-    String PARSER_CHAR_DOLLAR = "parser.character.dollar";
-
-    /**
-     * Set the character (onlysingle byte UTF-8 supported at present) to use instead of '#' for directives, macros and comments.
-     * @since 2.2
-     */
-    String PARSER_CHAR_HASH = "parser.character.hash";
-
-    /**
-     * Set the character (onlysingle byte UTF-8 supported at present) to use instead of '@' for '#@' block macros.
-     * @since 2.2
-     */
-    String PARSER_CHAR_AT = "parser.character.at";
-
-    /**
-     * Set the character (onlysingle byte UTF-8 supported at present) to use instead of '*' for '#* *#' block comments.
-     * @since 2.2
-     */
-    String PARSER_CHAR_ASTERISK = "parser.character.asterisk";
-
     /*
      * ----------------------------------------------------------------------
      * G E N E R A L  R U N T I M E  C O N F I G U R A T I O N
diff --git a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/RuntimeInstance.java b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/RuntimeInstance.java
index 929962b9..b3e0923e 100644
--- a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/RuntimeInstance.java
+++ b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/RuntimeInstance.java
@@ -62,6 +62,8 @@ import java.io.InputStream;
 import java.io.Reader;
 import java.io.StringReader;
 import java.io.Writer;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.charset.StandardCharsets;
 import java.util.Enumeration;
 import java.util.HashMap;
@@ -228,6 +230,12 @@ public class RuntimeInstance implements RuntimeConstants, RuntimeServices
     private LogContext logContext;
 
     /**
+     * Configured parser class
+     * @since 2.2
+     */
+    private Constructor parserConstructor;
+
+    /**
      * Configured '$' character
      * @since 2.2
      */
@@ -413,10 +421,6 @@ public class RuntimeInstance implements RuntimeConstants, RuntimeServices
 
         /* init parser behavior */
         hyphenAllowedInIdentifiers = getBoolean(PARSER_HYPHEN_ALLOWED, false);
-        dollar = getConfiguredCharacter(PARSER_CHAR_DOLLAR, '$');
-        hash = getConfiguredCharacter(PARSER_CHAR_HASH, '#');
-        at = getConfiguredCharacter(PARSER_CHAR_AT, '@');
-        asterisk = getConfiguredCharacter(PARSER_CHAR_ASTERISK, '*');
     }
 
     private char getConfiguredCharacter(String configKey, char defaultChar)
@@ -1191,6 +1195,29 @@ public class RuntimeInstance implements RuntimeConstants, RuntimeServices
     private void initializeParserPool()
     {
         /*
+         * First initialize parser class. If it's not valid or not found, it will generate an error
+         * later on in this method when parser creation is tester.
+         */
+        String parserClassName = getString(PARSER_CLASS, DEFAULT_PARSER_CLASS);
+        Class parserClass;
+        try
+        {
+            parserClass = ClassUtils.getClass(parserClassName);
+        }
+        catch (ClassNotFoundException cnfe)
+        {
+            throw new VelocityException("parser class not found: " + parserClassName, cnfe);
+        }
+        try
+        {
+            parserConstructor = parserClass.getConstructor(RuntimeServices.class);
+        }
+        catch (NoSuchMethodException nsme)
+        {
+            throw new VelocityException("parser class must provide a constructor taking a RuntimeServices argument", nsme);
+        }
+
+        /*
          * Which parser pool?
          */
         String pp = getString(RuntimeConstants.PARSER_POOL_CLASS);
@@ -1239,6 +1266,16 @@ public class RuntimeInstance implements RuntimeConstants, RuntimeServices
             parserPool = (ParserPool) o;
 
             parserPool.initialize(this);
+
+            /*
+             * test parser creation and use generated parser to fill up customized characters
+             */
+            Parser parser = parserPool.get();
+            dollar = parser.dollar();
+            hash = parser.hash();
+            at = parser.at();
+            asterisk = parser.asterisk();
+            parserPool.put(parser);
         }
         else
         {
@@ -1264,8 +1301,14 @@ public class RuntimeInstance implements RuntimeConstants, RuntimeServices
     public Parser createNewParser()
     {
         requireInitialization();
-
-        return new StandardParser(this);
+        try
+        {
+            return (Parser)parserConstructor.newInstance((RuntimeServices)this);
+        }
+        catch (IllegalAccessException | InstantiationException | InvocationTargetException e)
+        {
+            throw new VelocityException("could not build new parser class", e);
+        }
     }
 
     /**
diff --git a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/ParserConstants.java b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/ParserConstants.java
deleted file mode 100644
index b48383fe..00000000
--- a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/ParserConstants.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.apache.velocity.runtime.parser;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-public interface ParserConstants extends StandardParserConstants
-{
-}
diff --git a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/ParserTokenManager.java b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/ParserTokenManager.java
deleted file mode 100644
index 4074c982..00000000
--- a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/ParserTokenManager.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.apache.velocity.runtime.parser;
-
-public interface ParserTokenManager
-{
-    void clearStateVars();
-    void switchTo(int lexState);
-    int getCurrentLexicalState();
-    boolean stateStackPop();
-    boolean stateStackPush();
-    public Token getNextToken();
-    void setDebugStream(java.io.PrintStream ds);
-    boolean isInSet();
-    void setInSet(boolean value);
-    void ReInit(CharStream charStream);
-}
diff --git a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/node/ASTDirective.java b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/node/ASTDirective.java
index 25bcd62a..a8bf65e2 100644
--- a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/node/ASTDirective.java
+++ b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/node/ASTDirective.java
@@ -31,7 +31,7 @@ import org.apache.velocity.runtime.directive.Directive;
 import org.apache.velocity.runtime.directive.RuntimeMacro;
 import org.apache.velocity.runtime.parser.ParseException;
 import org.apache.velocity.runtime.parser.Parser;
-import org.apache.velocity.runtime.parser.ParserConstants;
+import org.apache.velocity.runtime.parser.StandardParserConstants;
 import org.apache.velocity.runtime.parser.Token;
 import org.apache.velocity.util.introspection.Info;
 
@@ -143,7 +143,7 @@ public class ASTDirective extends SimpleNode
                 }
 
                 t = getFirstToken();
-                if (t.kind == ParserConstants.WHITESPACE) t = t.next;
+                if (t.kind == StandardParserConstants.WHITESPACE) t = t.next;
                 directive.setLocation(t.beginLine, t.beginColumn, getTemplate());
                 directive.init(rsvc, context, this);
             }
diff --git a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/node/NodeUtils.java b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/node/NodeUtils.java
index 1146e736..84b735bd 100644
--- a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/node/NodeUtils.java
+++ b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/node/NodeUtils.java
@@ -20,7 +20,7 @@ package org.apache.velocity.runtime.parser.node;
  */
 
 import org.apache.velocity.runtime.parser.Parser;
-import org.apache.velocity.runtime.parser.ParserConstants;
+import org.apache.velocity.runtime.parser.StandardParserConstants;
 import org.apache.velocity.runtime.parser.Token;
 
 /**
@@ -130,7 +130,7 @@ public class NodeUtils
     public static String tokenLiteral( Parser parser, Token t )
     {
         // Look at kind of token and return "" when it's a block comment
-        if (t.kind == ParserConstants.MULTI_LINE_COMMENT)
+        if (t.kind == StandardParserConstants.MULTI_LINE_COMMENT)
         {
             return "";
         }
diff --git a/velocity-engine-core/src/main/parser/StandardParser.jjt b/velocity-engine-core/src/main/parser/Parser.jjt
index ffb44384..ecfef164 100644
--- a/velocity-engine-core/src/main/parser/StandardParser.jjt
+++ b/velocity-engine-core/src/main/parser/Parser.jjt
@@ -78,7 +78,7 @@ options
 
 }
 
-PARSER_BEGIN(StandardParser)
+PARSER_BEGIN(${parser.name}Parser)
 package org.apache.velocity.runtime.parser;
 
 import java.io.*;
@@ -106,9 +106,9 @@ import org.slf4j.Logger;
  * @author <a href="mailto:jvanzyl@apache.org">Jason van Zyl</a>
  * @author <a href="mailto:geirm@optonline.net">Geir Magnusson Jr.</a>
  * @author <a href="hps@intermeta.de">Henning P. Schmiedehausen</a>
- * @version $Id$
+ * @version $Id: $
 */
-public class StandardParser implements Parser
+public class ${parser.name}Parser implements Parser
 {
     /**
      * Parser debugging flag.
@@ -121,7 +121,7 @@ public class StandardParser implements Parser
     {
         try
         {
-            StandardParser.class.getDeclaredMethod("trace_call", String.class);
+            ${parser.name}Parser.class.getDeclaredMethod("trace_call", String.class);
             debugParser = true;
         }
         catch(NoSuchMethodException nsfe)
@@ -180,7 +180,7 @@ public class StandardParser implements Parser
      * object, we satisfy the requirement of an InputStream
      * by using a newline character as an input stream.
      */
-    public StandardParser( RuntimeServices rs)
+    public ${parser.name}Parser( RuntimeServices rs)
     {
         /*
          * need to call the CTOR first thing.
@@ -217,10 +217,10 @@ public class StandardParser implements Parser
         /*
          * then initialize customizable characters
          */
-         dollar = rsvc.dollar();
-         hash = rsvc.hash();
-         at = rsvc.at();
-         asterisk = rsvc.asterisk();
+         dollar   = '${parser.char.dollar}';
+         hash     = '${parser.char.hash}';
+         at       = '${parser.char.at}';
+         asterisk = '${parser.char.asterisk}';
     }
 
     /**
@@ -549,7 +549,7 @@ public class StandardParser implements Parser
     private char asterisk = '*';
 }
 
-PARSER_END(StandardParser)
+PARSER_END(${parser.name}Parser)
 
 TOKEN_MGR_DECLS:
 {
@@ -569,7 +569,7 @@ TOKEN_MGR_DECLS:
      */
     public static void trace(String message)
     {
-        StandardParser.trace(message);
+        ${parser.name}Parser.trace(message);
     }
     
     /**
@@ -906,7 +906,7 @@ TOKEN:
      *  for disaster, another long night with Mr. Parser, or both.
      */
 
-    <ESCAPE_DIRECTIVE :  (<DOUBLE_ESCAPE>)* "\\#" (<WORD> | <BRACKETED_WORD>) >
+    <ESCAPE_DIRECTIVE :  (<DOUBLE_ESCAPE>)* "\\${parser.char.hash}" (<WORD> | <BRACKETED_WORD>) >
 }
 
 
@@ -925,7 +925,7 @@ TOKEN:
 <DEFAULT, PRE_REFERENCE, PRE_OLD_REFERENCE, REFERENCE, REFMODIFIER, OLD_REFMODIFIER, REFMOD2, REFMOD3>
 TOKEN:
 {
-  <SET_DIRECTIVE: ("#set" | "#{set}")  (" "|"\t")* "(">
+  <SET_DIRECTIVE: ("${parser.char.hash}set" | "${parser.char.hash}{set}")  (" "|"\t")* "(">
     {
         if (! inComment)
         {
@@ -962,7 +962,7 @@ MORE :
      *   Note : DOLLARBANG is a duplicate of DOLLAR.  They must be identical.
      */
 
-    <DOLLAR: ("\\")* "$">
+    <DOLLAR: ("\\")* "${parser.char.dollar}">
     {
         if (! inComment)
         {
@@ -989,7 +989,7 @@ MORE :
         }
     }
 
-|   <DOLLARBANG: ("\\")* "$" ("\\")* "!">
+|   <DOLLARBANG: ("\\")* "${parser.char.dollar}" ("\\")* "!">
     {
         if (! inComment)
         {
@@ -1016,7 +1016,7 @@ MORE :
         }
     }
 
-|   "#[["
+|   "${parser.char.hash}[["
     {
        if (!inComment)
        {
@@ -1030,7 +1030,7 @@ MORE :
        }
     }
 
-|   <"#**" ~["#","\u200B"]>
+|   <"${parser.char.hash}${parser.char.asterisk}${parser.char.asterisk}" ~["${parser.char.hash}","\u200B"]>
     {
     	if (!inComment)
     	{
@@ -1045,7 +1045,7 @@ MORE :
     	}
     }
 
-|   "#*"
+|   "${parser.char.hash}${parser.char.asterisk}"
     {
     	if (!inComment)
     	{
@@ -1059,7 +1059,7 @@ MORE :
     	}
     }
 
-|   <HASH : "#" >
+|   <HASH : "${parser.char.hash}" >
     {
         if (! inComment)
         {
@@ -1092,7 +1092,7 @@ MORE :
 <DEFAULT,PRE_DIRECTIVE,DIRECTIVE,REFERENCE,PRE_REFERENCE,PRE_OLD_REFERENCE,REFMOD2,REFMOD3,REFMODIFIER,OLD_REFMODIFIER>
 TOKEN :
 {
-   <SINGLE_LINE_COMMENT_START: "##">
+   <SINGLE_LINE_COMMENT_START: "${parser.char.hash}${parser.char.hash}">
    {
         if (!inComment)
         {
@@ -1132,7 +1132,7 @@ TOKEN :
 <IN_FORMAL_COMMENT>
 TOKEN :
 {
-  <FORMAL_COMMENT: "*#" >
+  <FORMAL_COMMENT: "${parser.char.asterisk}${parser.char.hash}" >
   {
     inComment = false;
     stateStackPop();
@@ -1147,7 +1147,7 @@ TOKEN :
 <IN_MULTI_LINE_COMMENT>
 TOKEN :
 {
-  <MULTI_LINE_COMMENT: "*#" >
+  <MULTI_LINE_COMMENT: "${parser.char.asterisk}${parser.char.hash}" >
   {
     inComment = false;
     stateStackPop();
@@ -1162,7 +1162,7 @@ TOKEN :
 <IN_TEXTBLOCK>
 TOKEN :
 {
-  <TEXTBLOCK: "]]#" >
+  <TEXTBLOCK: "]]${parser.char.hash}" >
   {
     inComment = false;
     stateStackPop();
@@ -1375,8 +1375,8 @@ TOKEN:
 {
     <#LETTER: [ "a"-"z", "A"-"Z" ] >
 |   <#DIRECTIVE_CHAR: [ "a"-"z", "A"-"Z", "0"-"9", "_" ] >
-|   <WORD: ( <LETTER> | ["_"] | ["@"]) (<DIRECTIVE_CHAR>)* >
-|   <BRACKETED_WORD: "{" ( <LETTER> | ["_"] | ["@"]) (<DIRECTIVE_CHAR>)* "}" >
+|   <WORD: ( <LETTER> | ["_"] | ["${parser.char.at}"]) (<DIRECTIVE_CHAR>)* >
+|   <BRACKETED_WORD: "{" ( <LETTER> | ["_"] | ["${parser.char.at}"]) (<DIRECTIVE_CHAR>)* "}" >
 }
 
 /* -----------------------------------------------------------------------
@@ -1525,12 +1525,12 @@ TOKEN :
 {
     <DOUBLE_ESCAPE : "\\\\">
 |   <ESCAPE: "\\" >
-|   <TEXT: (~["$", "#", "\\", "\r", "\n","\u200B"])* (~["$", "#", "\\", "\r", "\n", " ", "\t","\u200B"])+ (~["$", "#", "\\", "\r", "\n","\u200B"])* <NEWLINE> ((~["$", "#", "\\", "\r", "\n","\u200B"])* <NEWLINE>)* >
+|   <TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])* (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n", " ", "\t","\u200B"])+ (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])* <NEWLINE> ((~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])* <NEWLINE>)* >
 }
 
 TOKEN :
 {
-    <INLINE_TEXT: (~["$", "#", "\\", "\r", "\n","\u200B"])+ >
+    <INLINE_TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])+ >
 }
 
 /**
@@ -1790,7 +1790,7 @@ boolean Directive(boolean afterNewline) :
     {
         String directiveName;
         int p = id.image.lastIndexOf(hash);
-        if (id.kind == ParserConstants.BRACKETED_WORD)
+        if (id.kind == StandardParserConstants.BRACKETED_WORD)
         {
             directiveName = id.image.substring(p + 2, id.image.length() - 1);
         }