|
1
19
20 package org.netbeans.modules.languages.lexer;
21
22 import java.util.Iterator  ;
23 import java.util.LinkedList ;
24 import java.util.List ;
25 import java.util.Map ;
26 import org.netbeans.api.languages.CharInput;
27 import org.netbeans.api.languages.ASTToken;
28 import org.netbeans.api.lexer.PartType;
29 import org.netbeans.api.languages.LanguagesManager;
30 import org.netbeans.api.languages.ParseException;
31 import org.netbeans.api.lexer.Token;
32 import org.netbeans.api.languages.ASTToken;
33 import org.netbeans.modules.languages.Feature;
34 import org.netbeans.modules.languages.Feature.Type;
35 import org.netbeans.modules.languages.Language;
36 import org.netbeans.modules.languages.LanguagesManagerImpl;
37 import org.netbeans.modules.languages.parser.Pattern;
38 import org.netbeans.spi.lexer.Lexer;
39 import org.netbeans.spi.lexer.LexerInput;
40 import org.netbeans.spi.lexer.LexerRestartInfo;
41 import org.netbeans.spi.lexer.TokenFactory;
42 import org.netbeans.modules.languages.parser.Parser;
43 import org.netbeans.spi.lexer.TokenPropertyProvider;
44
45
46
50 public class SLexer implements Lexer<STokenId>, Parser.Cookie {
51
52 private Language language;
53 private CharInput input;
54 private TokenFactory<STokenId> tokenFactory;
55 private Map <String ,STokenId> tokensMap;
56 private Parser parser;
57 private Object state;
58
59
60 SLexer (
61 Language language,
62 Map <String ,STokenId> tokensMap,
63 LexerRestartInfo<STokenId> info
64 ) {
65 this.language = language;
66 this.tokenFactory = info.tokenFactory ();
67 this.tokensMap = tokensMap;
68 this.state = info.state ();
69 parser = language.getParser ();
70 String outerMimeType = info.languagePath ().language (0).mimeType ();
71 try {
72 Language outerLanguage = ((LanguagesManagerImpl) LanguagesManager.getDefault ()).
73 getLanguage (outerMimeType);
74 this.input = createInputBridge (info.input (), outerLanguage);
75 } catch (ParseException ex) {
76 this.input = createInputBridge (info.input (), new Language (outerMimeType));
77 }
78 }
79
80 public Token<STokenId> nextToken () {
81 Token<STokenId> t = nextTokenIn ();
82 return t;
87 }
88
89 private Token<STokenId> nextTokenIn () {
90 if (state instanceof Marenka) {
91 return createToken ((Marenka) state);
92 }
93 int index = input.getIndex ();
94 if (input.eof ())
95 return createToken (index);
96 ASTToken token = null;
97 token = parser.read (this, input, language.getMimeType ());
98 if (language != null &&
99 tokenProperties != null &&
100 tokenProperties.getType ("call") != Type.NOT_SET
101 ) {
102 input.setIndex (index);
103 Object [] r = (Object []) tokenProperties.getValue ("call", new Object [] {input});
104 token = (ASTToken) r [0];
105 if (r [1] != null)
106 setState (((Integer ) r [1]).intValue ());
107 }
108
109 if (token == null) {
110 if (input.getIndex () > (index + 1))
111 input.setIndex (index + 1);
112 else
113 if (input.getIndex () == index)
114 input.read ();
115 return createToken ("error", index);
116 }
117 return createToken (token.getType (), index);
118 }
119
120 public Object state () {
121 return state;
122 }
123
124 public void release() {
125 }
126
127
128
130 private Feature tokenProperties;
131
132 public int getState () {
133 if (state == null) return -1;
134 return ((Integer ) state).intValue ();
135 }
136
137 public void setState (int state) {
138 this.state = new Integer (state);
139 }
140
141 public void setProperties (Feature tokenProperties) {
142 this.tokenProperties = tokenProperties;
143 }
144
145
146
148 private static CharInput createInputBridge (
149 LexerInput input,
150 Language language
151 ) {
152 Feature properties = language.getPreprocessorImport ();
153 if (properties != null) {
154 return new DelegatingInputBridge (
155 new InputBridge (input),
156 properties.getPattern ("start"),
157 properties.getPattern ("end"),
158 "PE" );
160 }
161 return new InputBridge (input);
162 }
163
164 private Token<STokenId> createToken (String type, int start) {
165 STokenId tokenId = tokensMap.get (type);
166 assert tokenId != null : "Unknown token type \"" + type + "\"";
167 if (!(input instanceof DelegatingInputBridge)) {
168 return tokenFactory.createToken (tokenId);
169 }
170 List embeddings = ((DelegatingInputBridge) input).getEmbeddings ();
171 if (embeddings.isEmpty ())
172 return tokenFactory.createToken (tokenId);
173 Map <String ,Feature> imports = language.getTokenImports ();
174 if (imports.containsKey (type))
175 return tokenFactory.createToken (tokenId);
177 Marenka marenka = new Marenka ((Integer ) state);
178 String property = "S";
179 Iterator it = embeddings.iterator ();
180 while(it.hasNext ()) {
181 Vojta v = (Vojta) it.next ();
182 if (start < v.startOffset) {
183 marenka.add (new Vojta (type, start, v.startOffset, property));
184 property = "C";
185 }
186 marenka.add (v);
187 start = v.endOffset;
188 }
189 if (start < input.getIndex ())
190 marenka.add (new Vojta (type, start, input.getIndex (), property));
191 return createToken (marenka);
192 }
193
194 private Token<STokenId> createToken (int start) {
195 if (!(input instanceof DelegatingInputBridge)) {
196 return null;
197 }
198 List embeddings = ((DelegatingInputBridge) input).getEmbeddings ();
199 if (embeddings.isEmpty ())
200 return null;
201 Marenka marenka = new Marenka ((Integer ) state);
202 String property = "S";
203 Iterator it = embeddings.iterator ();
204 while(it.hasNext ()) {
205 Vojta v = (Vojta) it.next ();
206 assert start == v.startOffset;
207 marenka.add (v);
208 start = v.endOffset;
209 }
210 assert start == input.getIndex ();
211 return createToken (marenka);
212 }
213
214 private Token<STokenId> createToken (Marenka marenka) {
215 Vojta v = marenka.removeFirst ();
216 STokenId tokenId = tokensMap.get (v.type);
217 assert tokenId != null : "Unknown type " + v.type;
218 input.setIndex (v.endOffset);
219 if (marenka.isEmpty ())
220 this.state = marenka.getState ();
221 else
222 this.state = marenka;
223 if (v.property instanceof TokenProperties)
225 return tokenFactory.createPropertyToken (
226 tokenId,
227 v.endOffset - v.startOffset,
228 (TokenProperties) v.property,
229 PartType.COMPLETE
230 );
231 else
232 return tokenFactory.createPropertyToken (
233 tokenId,
234 v.endOffset - v.startOffset,
235 new TokenPropProvider(v.property),
236 PartType.COMPLETE
237 );
238 }
239
240 private static String e (CharSequence t) {
241 StringBuilder sb = new StringBuilder ();
242 int i, k = t.length ();
243 for (i = 0; i < k; i++) {
244 if (t.charAt (i) == '\t')
245 sb.append ("\\t");
246 else
247 if (t.charAt (i) == '\r')
248 sb.append ("\\r");
249 else
250 if (t.charAt (i) == '\n')
251 sb.append ("\\n");
252 else
253 sb.append (t.charAt (i));
254 }
255 return sb.toString ();
256 }
257
258
259
261 private static final class TokenPropProvider implements TokenPropertyProvider {
262
263 private final Object value;
264
265 TokenPropProvider(Object value) {
266 this.value = value;
267 }
268
269 public Object getValue (Token token, Object key) {
270 if ("type".equals(key))
271 return value;
272 return null;
273 }
274
275 }
276
277 static class TokenProperties implements TokenPropertyProvider {
278
279 private String type;
280 private int startSkipLength;
281 private int endSkipLength;
282
283 TokenProperties (
284 String type,
285 int startSkipLength,
286 int endSkipLength
287 ) {
288 this.type = type;
289 this.startSkipLength = startSkipLength;
290 this.endSkipLength = endSkipLength;
291 }
292
293 public Object getValue (Token token, Object key) {
294 if ("type".equals (key)) return type;
295 if ("startSkipLength".equals (key)) return new Integer (startSkipLength);
296 if ("endSkipLength".equals (key)) return new Integer (endSkipLength);
297 return null;
298 }
299
300 };
301
302 static class Vojta {
303
304 String type;
305 int startOffset;
306 int endOffset;
307 Object property;
308
309 Vojta (
310 String type,
311 int startOffset,
312 int endOffset,
313 Object property
314 ) {
315 this.type = type;
316 this.startOffset = startOffset;
317 this.endOffset = endOffset;
318 this.property = property;
319 }
320
321 int size () {
322 return endOffset - startOffset;
323 }
324 }
325
326 static class Marenka {
327
328 Integer state;
329 LinkedList <Vojta> vojta = new LinkedList <Vojta> ();
330
331 Marenka (Integer state) {
332 this.state = state;
333 }
334
335 void add (Vojta vojta) {
336 this.vojta.add (vojta);
337 }
338
339 Vojta removeFirst () {
340 return vojta.removeFirst ();
341 }
342
343 boolean isEmpty () {
344 return vojta.isEmpty ();
345 }
346
347 Integer getState () {
348 return state;
349 }
350 }
351 }
352
353
354 |
Popular Tags |
Java API By Example, From Geeks To Geeks.