|
1
2
3 package org.sablecc.sablecc.launcher.syntax3.lexer;
4
5 import java.io.*;
6 import org.sablecc.sablecc.launcher.syntax3.node.*;
7
8 @SuppressWarnings  ("nls")
9 public class Lexer
10 {
11 protected Token token;
12 protected State state = State.START;
13
14 private PushbackReader in;
15 private int line;
16 private int pos;
17 private boolean cr;
18 private boolean eof;
19 private final StringBuffer text = new StringBuffer ();
20
21 @SuppressWarnings ("unused")
22 protected void filter() throws LexerException, IOException
23 {
24 }
26
27 public Lexer(@SuppressWarnings ("hiding") PushbackReader in)
28 {
29 this.in = in;
30 }
31
32 public Token peek() throws LexerException, IOException
33 {
34 while(this.token == null)
35 {
36 this.token = getToken();
37 filter();
38 }
39
40 return this.token;
41 }
42
43 public Token next() throws LexerException, IOException
44 {
45 while(this.token == null)
46 {
47 this.token = getToken();
48 filter();
49 }
50
51 Token result = this.token;
52 this.token = null;
53 return result;
54 }
55
56 protected Token getToken() throws IOException, LexerException
57 {
58 int dfa_state = 0;
59
60 int start_pos = this.pos;
61 int start_line = this.line;
62
63 int accept_state = -1;
64 int accept_token = -1;
65 int accept_length = -1;
66 int accept_pos = -1;
67 int accept_line = -1;
68
69 @SuppressWarnings ("hiding") int[][][] gotoTable = Lexer.gotoTable[this.state.id()];
70 @SuppressWarnings ("hiding") int[] accept = Lexer.accept[this.state.id()];
71 this.text.setLength(0);
72
73 while(true)
74 {
75 int c = getChar();
76
77 if(c != -1)
78 {
79 switch(c)
80 {
81 case 10:
82 if(this.cr)
83 {
84 this.cr = false;
85 }
86 else
87 {
88 this.line++;
89 this.pos = 0;
90 }
91 break;
92 case 13:
93 this.line++;
94 this.pos = 0;
95 this.cr = true;
96 break;
97 default:
98 this.pos++;
99 this.cr = false;
100 break;
101 }
102
103 this.text.append((char) c);
104
105 do
106 {
107 int oldState = (dfa_state < -1) ? (-2 -dfa_state) : dfa_state;
108
109 dfa_state = -1;
110
111 int[][] tmp1 = gotoTable[oldState];
112 int low = 0;
113 int high = tmp1.length - 1;
114
115 while(low <= high)
116 {
117 int middle = (low + high) / 2;
118 int[] tmp2 = tmp1[middle];
119
120 if(c < tmp2[0])
121 {
122 high = middle - 1;
123 }
124 else if(c > tmp2[1])
125 {
126 low = middle + 1;
127 }
128 else
129 {
130 dfa_state = tmp2[2];
131 break;
132 }
133 }
134 }while(dfa_state < -1);
135 }
136 else
137 {
138 dfa_state = -1;
139 }
140
141 if(dfa_state >= 0)
142 {
143 if(accept[dfa_state] != -1)
144 {
145 accept_state = dfa_state;
146 accept_token = accept[dfa_state];
147 accept_length = this.text.length();
148 accept_pos = this.pos;
149 accept_line = this.line;
150 }
151 }
152 else
153 {
154 if(accept_state != -1)
155 {
156 switch(accept_token)
157 {
158 case 0:
159 {
160 @SuppressWarnings ("hiding") Token token = new0(
161 start_line + 1,
162 start_pos + 1);
163 pushBack(accept_length);
164 this.pos = accept_pos;
165 this.line = accept_line;
166 switch(state.id())
167 {
168 case 0: state = State.SHORT_OPTION; break;
169 }
170 return token;
171 }
172 case 1:
173 {
174 @SuppressWarnings ("hiding") Token token = new1(
175 start_line + 1,
176 start_pos + 1);
177 pushBack(accept_length);
178 this.pos = accept_pos;
179 this.line = accept_line;
180 switch(state.id())
181 {
182 case 0: state = State.LONG_OPTION; break;
183 }
184 return token;
185 }
186 case 2:
187 {
188 @SuppressWarnings ("hiding") Token token = new2(
189 getText(accept_length),
190 start_line + 1,
191 start_pos + 1);
192 pushBack(accept_length);
193 this.pos = accept_pos;
194 this.line = accept_line;
195 switch(state.id())
196 {
197 case 0: state = State.START; break;
198 }
199 return token;
200 }
201 case 3:
202 {
203 @SuppressWarnings ("hiding") Token token = new3(
204 getText(accept_length),
205 start_line + 1,
206 start_pos + 1);
207 pushBack(accept_length);
208 this.pos = accept_pos;
209 this.line = accept_line;
210 switch(state.id())
211 {
212 case 1: state = State.SHORT_OPTION; break;
213 }
214 return token;
215 }
216 case 4:
217 {
218 @SuppressWarnings ("hiding") Token token = new4(
219 getText(accept_length),
220 start_line + 1,
221 start_pos + 1);
222 pushBack(accept_length);
223 this.pos = accept_pos;
224 this.line = accept_line;
225 switch(state.id())
226 {
227 case 2: state = State.LONG_OPTION; break;
228 }
229 return token;
230 }
231 case 5:
232 {
233 @SuppressWarnings ("hiding") Token token = new5(
234 start_line + 1,
235 start_pos + 1);
236 pushBack(accept_length);
237 this.pos = accept_pos;
238 this.line = accept_line;
239 switch(state.id())
240 {
241 case 2: state = State.OPERAND; break;
242 case 1: state = State.OPERAND; break;
243 }
244 return token;
245 }
246 case 6:
247 {
248 @SuppressWarnings ("hiding") Token token = new6(
249 getText(accept_length),
250 start_line + 1,
251 start_pos + 1);
252 pushBack(accept_length);
253 this.pos = accept_pos;
254 this.line = accept_line;
255 switch(state.id())
256 {
257 case 3: state = State.OPERAND; break;
258 }
259 return token;
260 }
261 }
262 }
263 else
264 {
265 if(this.text.length() > 0)
266 {
267 throw new LexerException(
268 "[" + (start_line + 1) + "," + (start_pos + 1) + "]" +
269 " Unknown token: " + this.text);
270 }
271
272 @SuppressWarnings ("hiding") EOF token = new EOF(
273 start_line + 1,
274 start_pos + 1);
275 return token;
276 }
277 }
278 }
279 }
280
281 Token new0(@SuppressWarnings ("hiding") int line, @SuppressWarnings ("hiding") int pos) { return new THyphen(line, pos); }
282 Token new1(@SuppressWarnings ("hiding") int line, @SuppressWarnings ("hiding") int pos) { return new TDoubleHyphen(line, pos); }
283 Token new2(@SuppressWarnings ("hiding") String text, @SuppressWarnings ("hiding") int line, @SuppressWarnings ("hiding") int pos) { return new TText(text, line, pos); }
284 Token new3(@SuppressWarnings ("hiding") String text, @SuppressWarnings ("hiding") int line, @SuppressWarnings ("hiding") int pos) { return new TShortName(text, line, pos); }
285 Token new4(@SuppressWarnings ("hiding") String text, @SuppressWarnings ("hiding") int line, @SuppressWarnings ("hiding") int pos) { return new TLongName(text, line, pos); }
286 Token new5(@SuppressWarnings ("hiding") int line, @SuppressWarnings ("hiding") int pos) { return new TAssign(line, pos); }
287 Token new6(@SuppressWarnings ("hiding") String text, @SuppressWarnings ("hiding") int line, @SuppressWarnings ("hiding") int pos) { return new TOperandText(text, line, pos); }
288
289 private int getChar() throws IOException
290 {
291 if(this.eof)
292 {
293 return -1;
294 }
295
296 int result = this.in.read();
297
298 if(result == -1)
299 {
300 this.eof = true;
301 }
302
303 return result;
304 }
305
306 private void pushBack(int acceptLength) throws IOException
307 {
308 int length = this.text.length();
309 for(int i = length - 1; i >= acceptLength; i--)
310 {
311 this.eof = false;
312
313 this.in.unread(this.text.charAt(i));
314 }
315 }
316
317 protected void unread(@SuppressWarnings ("hiding") Token token) throws IOException
318 {
319 @SuppressWarnings ("hiding") String text = token.getText();
320 int length = text.length();
321
322 for(int i = length - 1; i >= 0; i--)
323 {
324 this.eof = false;
325
326 this.in.unread(text.charAt(i));
327 }
328
329 this.pos = token.getPos() - 1;
330 this.line = token.getLine() - 1;
331 }
332
333 private String getText(int acceptLength)
334 {
335 StringBuffer s = new StringBuffer (acceptLength);
336 for(int i = 0; i < acceptLength; i++)
337 {
338 s.append(this.text.charAt(i));
339 }
340
341 return s.toString();
342 }
343
344 private static int[][][][] gotoTable;
345
374
375 private static int[][] accept;
376
387
388 public static class State
389 {
390 public final static State START = new State(0);
391 public final static State SHORT_OPTION = new State(1);
392 public final static State LONG_OPTION = new State(2);
393 public final static State OPERAND = new State(3);
394
395 private int id;
396
397 private State(@SuppressWarnings ("hiding") int id)
398 {
399 this.id = id;
400 }
401
402 public int id()
403 {
404 return this.id;
405 }
406 }
407
408 static
409 {
410 try
411 {
412 DataInputStream s = new DataInputStream(
413 new BufferedInputStream(
414 Lexer.class.getResourceAsStream("lexer.dat")));
415
416 int length = s.readInt();
418 gotoTable = new int[length][][][];
419 for(int i = 0; i < gotoTable.length; i++)
420 {
421 length = s.readInt();
422 gotoTable[i] = new int[length][][];
423 for(int j = 0; j < gotoTable[i].length; j++)
424 {
425 length = s.readInt();
426 gotoTable[i][j] = new int[length][3];
427 for(int k = 0; k < gotoTable[i][j].length; k++)
428 {
429 for(int l = 0; l < 3; l++)
430 {
431 gotoTable[i][j][k][l] = s.readInt();
432 }
433 }
434 }
435 }
436
437 length = s.readInt();
439 accept = new int[length][];
440 for(int i = 0; i < accept.length; i++)
441 {
442 length = s.readInt();
443 accept[i] = new int[length];
444 for(int j = 0; j < accept[i].length; j++)
445 {
446 accept[i][j] = s.readInt();
447 }
448 }
449
450 s.close();
451 }
452 catch(Exception e)
453 {
454 throw new RuntimeException ("The file \"lexer.dat\" is either missing or corrupted.");
455 }
456 }
457 }
458 |
Popular Tags |
Java API By Example, From Geeks To Geeks.