diff options
3 files changed, 18 insertions, 10 deletions
diff --git a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java index f6a93c08..e6a06151 100644 --- a/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java +++ b/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java @@ -61,9 +61,11 @@ implements CharStream private int inBuf = 0; /* CB - to properly handle EOF *inside* javacc lexer, - * we send a 'zero-width whitespace' *just before* EOF + * we send a 'file separator' ascii char *just before* EOF + * (see https://en.wikipedia.org/wiki/Delimiter#ASCII_delimited_text) */ private boolean beforeEOF = false; + private static char END_OF_FILE = '\u001C'; private void ExpandBuff(boolean wrapAround) { @@ -161,7 +163,7 @@ implements CharStream inputStream.close(); throw new java.io.IOException(); } - buffer[maxNextCharInd++] = '\u200B'; + buffer[maxNextCharInd++] = END_OF_FILE; beforeEOF = true; } else diff --git a/velocity-engine-core/src/main/parser/Parser.jjt b/velocity-engine-core/src/main/parser/Parser.jjt index 3fbfe28b..16a1dae7 100644 --- a/velocity-engine-core/src/main/parser/Parser.jjt +++ b/velocity-engine-core/src/main/parser/Parser.jjt @@ -749,7 +749,7 @@ TOKEN_MGR_DECLS: <PRE_DIRECTIVE,PRE_REFERENCE,PRE_OLD_REFERENCE> TOKEN : { - <LONE_SYMBOL: "\u200B" > + <LONE_SYMBOL: "\u001C" > { stateStackPop(); } @@ -759,7 +759,7 @@ TOKEN : <REFERENCE,REFMODIFIER,OLD_REFMODIFIER,REFMOD3,REFINDEX,DIRECTIVE,REFMOD2,DEFAULT,REFMOD,IN_TEXTBLOCK,IN_MULTILINE_COMMENT,IN_FORMAL_COMMENT,IN_SINGLE_LINE_COMMENT> TOKEN : { - <ZERO_WIDTH_WHITESPACE: "\u200B"> + <ZERO_WIDTH_WHITESPACE: "\u001C"> } <REFERENCE, REFMODIFIER, OLD_REFMODIFIER, REFMOD3> @@ -1031,7 +1031,7 @@ MORE : } } -| <"${parser.char.hash}${parser.char.asterisk}${parser.char.asterisk}" ~["${parser.char.hash}","\u200B"]> +| <"${parser.char.hash}${parser.char.asterisk}${parser.char.asterisk}" ~["${parser.char.hash}","\u001C"]> { if (!inComment) { @@ -1179,7 +1179,7 @@ SKIP : <IN_TEXTBLOCK> MORE : { - < ~["\u200B"] > + < ~["\u001C"] > } /* ----------------------------------------------------------------------- @@ -1219,7 +1219,7 @@ TOKEN : // <STRING_LITERAL: ( "\"" ( ~["\"","\n","\r"] )* "\"" ) | ( "'" ( ~["'","\n","\r"] )* "'" ) > < STRING_LITERAL: ("\"" - ( (~["\"","\u200B"]) + ( (~["\"","\u001C"]) | ("\\" ( ["n","t","b","r","f"] | ["0"-"7"] ( ["0"-"7"] )? @@ -1234,7 +1234,7 @@ TOKEN : ) | ("\'" - ( (~["\'","\u200B"]) + ( (~["\'","\u001C"]) | ("''") | ( "\\" (" ")* "\n") )* @@ -1526,12 +1526,12 @@ TOKEN : { <DOUBLE_ESCAPE : "\\\\"> | <ESCAPE: "\\" > -| <TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])* (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n", " ", "\t","\u200B"])+ (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])* <NEWLINE> ((~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])* <NEWLINE>)* > +| <TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u001C"])* (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n", " ", "\t","\u001C"])+ (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u001C"])* <NEWLINE> ((~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u001C"])* <NEWLINE>)* > } TOKEN : { - <INLINE_TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])+ > + <INLINE_TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u001C"])+ > } /** diff --git a/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java b/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java index 1f2d3559..2ed509e5 100644 --- a/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java +++ b/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java @@ -28,4 +28,10 @@ public class Velocity919TestCase extends BaseTestCase assertEvalEquals("before\u200Bafter", "before\u200Bafter"); } + public void testUserFileSeparator() throws Exception + { + assertEvalEquals("before\u001Cafter", "before\u001Cafter"); + } + } + |