1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.lexer;
import com.intellij.lang.java.JavaParserDefinition;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.JavaTokenType;
import com.intellij.psi.tree.IElementType;
import junit.framework.TestCase;
/**
* @author max
*/
public class LayeredLexerTest extends TestCase {
public void testInTheMiddle() throws Exception {
Lexer lexer = setupLexer("s=\"abc\\ndef\";");
assertEquals("s", nextToken(lexer));
assertEquals("=", nextToken(lexer));
assertEquals("\"abc", nextToken(lexer));
assertEquals("\\n", nextToken(lexer));
assertEquals("def\"", nextToken(lexer));
assertEquals(";", nextToken(lexer));
assertEquals(null, lexer.getTokenType());
}
public void testModification() throws Exception {
Lexer lexer = setupLexer("s=\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\";");
assertEquals("s", nextToken(lexer));
assertEquals("=", nextToken(lexer));
assertEquals("\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"", nextToken(lexer));
assertEquals(";", nextToken(lexer));
assertEquals(null, lexer.getTokenType());
lexer.start(lexer.getBufferSequence(), 2, lexer.getBufferEnd(), (short) 1);
assertEquals("\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"", nextToken(lexer));
assertEquals(";", nextToken(lexer));
assertEquals(null, lexer.getTokenType());
}
public void testInTheAtStartup() throws Exception {
Lexer lexer = setupLexer("s=\"\\ndef\";");
assertEquals("s", nextToken(lexer));
assertEquals("=", nextToken(lexer));
assertEquals("\"", nextToken(lexer));
assertEquals("\\n", nextToken(lexer));
assertEquals("def\"", nextToken(lexer));
assertEquals(";", nextToken(lexer));
assertEquals(null, lexer.getTokenType());
}
public void testInTheAtEnd() throws Exception {
Lexer lexer = setupLexer("s=\"abc\\n\";");
assertEquals("s", nextToken(lexer));
assertEquals("=", nextToken(lexer));
assertEquals("\"abc", nextToken(lexer));
assertEquals("\\n", nextToken(lexer));
assertEquals("\"", nextToken(lexer));
assertEquals(";", nextToken(lexer));
assertEquals(null, lexer.getTokenType());
}
public void testNonTerminated() throws Exception {
Lexer lexer = setupLexer("s=\"abc\\n");
assertEquals("s", nextToken(lexer));
assertEquals("=", nextToken(lexer));
assertEquals("\"abc", nextToken(lexer));
assertEquals("\\n", nextToken(lexer));
assertEquals(null, lexer.getTokenType());
}
public void testNonTerminated2() throws Exception {
Lexer lexer = setupLexer("s=\"abc\\");
assertEquals("s", nextToken(lexer));
assertEquals("=", nextToken(lexer));
assertEquals("\"abc", nextToken(lexer));
assertEquals("\\", nextToken(lexer));
assertEquals(null, lexer.getTokenType());
}
public void testUnicode() throws Exception {
Lexer lexer = setupLexer("s=\"\\uFFFF\";");
assertEquals("s", nextToken(lexer));
assertEquals("=", nextToken(lexer));
assertEquals("\"", nextToken(lexer));
assertEquals("\\uFFFF", nextToken(lexer));
assertEquals("\"", nextToken(lexer));
assertEquals(";", nextToken(lexer));
assertEquals(null, lexer.getTokenType());
}
private static Lexer setupLexer(String text) {
LayeredLexer lexer = new LayeredLexer(JavaParserDefinition.createLexer(LanguageLevel.JDK_1_3));
lexer.registerSelfStoppingLayer(new StringLiteralLexer('\"', JavaTokenType.STRING_LITERAL),
new IElementType[]{JavaTokenType.STRING_LITERAL},
IElementType.EMPTY_ARRAY);
lexer.start(text);
return lexer;
}
private static String nextToken(Lexer lexer) {
assertTrue(lexer.getTokenType() != null);
final String s = lexer.getBufferSequence().subSequence(lexer.getTokenStart(), lexer.getTokenEnd()).toString();
lexer.advance();
return s;
}
}
|
