1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
/*
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.lexer;
import com.intellij.psi.tree.IElementType;
import org.jetbrains.annotations.NotNull;
/**
* Naive implementation of lexer that able to combine element types of two other lexers.
*
* The implementation doesn't give any guarantees about valid points of 'incremental relexing',
* because it returns start/end-offsets not for token that was used but for the shortest one.
*
* Also it reduces state size to 16 bits for nested lexer.
*
* @deprecated use {@link com.intellij.psi.templateLanguages.TemplateBlackAndWhiteLexer} or {@link com.intellij.lexer.LayeredLexer} instead
*/
public abstract class CompositeLexer extends LexerBase {
private final Lexer myLexer1;
private final Lexer myLexer2;
private int myCurOffset;
public CompositeLexer(Lexer lexer1, Lexer lexer2) {
myLexer1 = lexer1;
myLexer2 = lexer2;
}
protected abstract IElementType getCompositeTokenType(IElementType type1, IElementType type2);
@Override
public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) {
myLexer1.start(buffer, startOffset, endOffset, (initialState >> 16) & 0xFFFF);
myLexer2.start(buffer, startOffset, endOffset, initialState & 0xFFFF);
myCurOffset = startOffset;
}
@NotNull
@Override
public CharSequence getBufferSequence() {
return myLexer1.getBufferSequence();
}
@Override
public int getState() {
final int state = myLexer1.getState();
final int state2 = myLexer2.getState();
if (state >= 0 && state < Short.MAX_VALUE &&
state2 >= 0 && state2 < Short.MAX_VALUE) {
return (state << 16) + state2;
}
return 0;
}
@Override
public IElementType getTokenType() {
IElementType type1 = myLexer1.getTokenType();
if (type1 == null) return null;
IElementType type2 = myLexer2.getTokenType();
return getCompositeTokenType(type1, type2);
}
@Override
public int getTokenStart() {
return myCurOffset;
}
@Override
public int getTokenEnd() {
return Math.min(myLexer1.getTokenEnd(), myLexer2.getTokenEnd());
}
@Override
public void advance() {
int end1 = myLexer1.getTokenEnd();
int end2 = myLexer2.getTokenEnd();
myCurOffset = Math.min(end1, end2);
if (myCurOffset == end1){
myLexer1.advance();
}
if (myCurOffset == end2){
myLexer2.advance();
}
}
@Override
public int getBufferEnd() {
return myLexer1.getBufferEnd();
}
}
|
