|
1
22
23 package org.javacc.parser;
24
25 import java.util.*;
26 import java.io.*;
27
28 public class LexGen
29 extends JavaCCGlobals
30 implements JavaCCParserConstants
31 {
32 static private java.io.PrintWriter  ostr;
33 static private String staticString;
34 static private String tokMgrClassName;
35
36 static Hashtable allTpsForState = new Hashtable();
38 public static int lexStateIndex = 0;
39 static int[] kinds;
40 public static int maxOrdinal = 1;
41 public static String lexStateSuffix;
42 static String [] newLexState;
43 public static int[] lexStates;
44 public static boolean[] ignoreCase;
45 public static Action[] actions;
46 public static Hashtable initStates = new Hashtable();
47 public static int stateSetSize;
48 public static int maxLexStates;
49 public static String [] lexStateName;
50 static NfaState[] singlesToSkip;
51 public static long[] toSkip;
52 public static long[] toSpecial;
53 public static long[] toMore;
54 public static long[] toToken;
55 public static int defaultLexState;
56 public static RegularExpression[] rexprs;
57 public static int[] maxLongsReqd;
58 public static int[] initMatch;
59 public static int[] canMatchAnyChar;
60 public static boolean hasEmptyMatch;
61 public static boolean[] canLoop;
62 public static boolean[] stateHasActions;
63 public static boolean hasLoop = false;
64 public static boolean[] canReachOnMore;
65 public static boolean[] hasNfa;
66 public static boolean[] mixed;
67 public static NfaState initialState;
68 public static int curKind;
69 static boolean hasSkipActions = false;
70 static boolean hasMoreActions = false;
71 static boolean hasTokenActions = false;
72 static boolean hasSpecial = false;
73 static boolean hasSkip = false;
74 static boolean hasMore = false;
75 static boolean hasToken = false;
76 public static RegularExpression curRE;
77 public static boolean keepLineCol;
78
79 static void PrintClassHead()
80 {
81 int i, j;
82
83 try {
84 File tmp = new File(Options.getOutputDirectory(), tokMgrClassName + ".java");
85 ostr = new java.io.PrintWriter (
86 new java.io.BufferedWriter (
87 new java.io.FileWriter (tmp),
88 8092
89 )
90 );
91 Vector tn = (Vector)(toolNames.clone());
92 tn.addElement(toolName);
93
94 ostr.println("/* " + getIdString(tn, tokMgrClassName + ".java") + " */");
95
96 int l = 0, kind;
97 i = 1;
98 for (;;)
99 {
100 if (cu_to_insertion_point_1.size() <= l)
101 break;
102
103 kind = ((Token)cu_to_insertion_point_1.elementAt(l)).kind;
104 if(kind == PACKAGE || kind == IMPORT) {
105 for (; i < cu_to_insertion_point_1.size(); i++) {
106 kind = ((Token)cu_to_insertion_point_1.elementAt(i)).kind;
107 if (kind == SEMICOLON ||
108 kind == ABSTRACT ||
109 kind == FINAL ||
110 kind == PUBLIC ||
111 kind == CLASS ||
112 kind == INTERFACE)
113 {
114 cline = ((Token)(cu_to_insertion_point_1.elementAt(l))).beginLine;
115 ccol = ((Token)(cu_to_insertion_point_1.elementAt(l))).beginColumn;
116 for (j = l; j < i; j++) {
117 printToken((Token)(cu_to_insertion_point_1.elementAt(j)), ostr);
118 }
119 if (kind == SEMICOLON)
120 printToken((Token)(cu_to_insertion_point_1.elementAt(j)), ostr);
121 ostr.println("");
122 break;
123 }
124 }
125 l = ++i;
126 }
127 else
128 break;
129 }
130
131 ostr.println("");
132 ostr.println("public class " + tokMgrClassName + " implements " +
133 cu_name + "Constants");
134 ostr.println("{"); }
136 catch (java.io.IOException err) {
137 JavaCCErrors.semantic_error("Could not create file : " + tokMgrClassName + ".java\n");
138 throw new Error ();
139 }
140
141 if (token_mgr_decls != null && token_mgr_decls.size() > 0)
142 {
143 Token t = (Token)token_mgr_decls.elementAt(0);
144 boolean commonTokenActionSeen = false;
145 boolean commonTokenActionNeeded = Options.getCommonTokenAction();
146
147 printTokenSetup((Token)token_mgr_decls.elementAt(0));
148 ccol = 1;
149
150 for (j = 0; j < token_mgr_decls.size(); j++)
151 {
152 t = (Token)token_mgr_decls.elementAt(j);
153 if (t.kind == IDENTIFIER &&
154 commonTokenActionNeeded &&
155 !commonTokenActionSeen)
156 commonTokenActionSeen = t.image.equals("CommonTokenAction");
157
158 printToken(t, ostr);
159 }
160
161 ostr.println("");
162 if (commonTokenActionNeeded && !commonTokenActionSeen)
163 JavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. But it appears you have not defined the method :\n"+
164 " " + staticString + "void CommonTokenAction(Token t)\n" +
165 "in your TOKEN_MGR_DECLS. The generated token manager will not compile.");
166
167 }
168 else if (Options.getCommonTokenAction())
169 {
170 JavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. But you have not defined the method :\n"+
171 " " + staticString + "void CommonTokenAction(Token t)\n" +
172 "in your TOKEN_MGR_DECLS. The generated token manager will not compile.");
173 }
174
175 ostr.println(" public " + staticString + " java.io.PrintStream debugStream = System.out;");
176 ostr.println(" public " + staticString + " void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }");
177
178 if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
179 ostr.println(" public " + cu_name + " parser = null;");
180 }
181 }
182
183 static void DumpDebugMethods()
184 {
185
186 ostr.println(" " + staticString + " int kindCnt = 0;");
187 ostr.println(" protected " + staticString + " final String jjKindsForBitVector(int i, long vec)");
188 ostr.println(" {");
189 ostr.println(" String retVal = \"\";");
190 ostr.println(" if (i == 0)");
191 ostr.println(" kindCnt = 0;");
192 ostr.println(" for (int j = 0; j < 64; j++)");
193 ostr.println(" {");
194 ostr.println(" if ((vec & (1L << j)) != 0L)");
195 ostr.println(" {");
196 ostr.println(" if (kindCnt++ > 0)");
197 ostr.println(" retVal += \", \";");
198 ostr.println(" if (kindCnt % 5 == 0)");
199 ostr.println(" retVal += \"\\n \";");
200 ostr.println(" retVal += tokenImage[i * 64 + j];");
201 ostr.println(" }");
202 ostr.println(" }");
203 ostr.println(" return retVal;");
204 ostr.println(" }");
205 ostr.println("");
206
207 ostr.println(" protected " + staticString + " final String jjKindsForStateVector(int lexState, int[] vec, int start, int end)");
208 ostr.println(" {");
209 ostr.println(" boolean[] kindDone = new boolean[" + maxOrdinal + "];");
210 ostr.println(" String retVal = \"\";");
211 ostr.println(" int cnt = 0;");
212 ostr.println(" for (int i = start; i < end; i++)");
213 ostr.println(" {");
214 ostr.println(" if (vec[i] == -1)");
215 ostr.println(" continue;");
216 ostr.println(" int[] stateSet = statesForState[curLexState][vec[i]];");
217 ostr.println(" for (int j = 0; j < stateSet.length; j++)");
218 ostr.println(" {");
219 ostr.println(" int state = stateSet[j];");
220 ostr.println(" if (!kindDone[kindForState[lexState][state]])");
221 ostr.println(" {");
222 ostr.println(" kindDone[kindForState[lexState][state]] = true;");
223 ostr.println(" if (cnt++ > 0)");
224 ostr.println(" retVal += \", \";");
225 ostr.println(" if (cnt % 5 == 0)");
226 ostr.println(" retVal += \"\\n \";");
227 ostr.println(" retVal += tokenImage[kindForState[lexState][state]];");
228 ostr.println(" }");
229 ostr.println(" }");
230 ostr.println(" }");
231 ostr.println(" if (cnt == 0)");
232 ostr.println(" return \"{ }\";");
233 ostr.println(" else");
234 ostr.println(" return \"{ \" + retVal + \" }\";");
235 ostr.println(" }");
236 ostr.println("");
237 }
238
239 static void BuildLexStatesTable()
240 {
241 Enumeration e = rexprlist.elements();
242 TokenProduction tp;
243 int i;
244
245 String [] tmpLexStateName = new String [lexstate_I2S.size()];
246 while (e.hasMoreElements())
247 {
248 tp = (TokenProduction)e.nextElement();
249 Vector respecs = tp.respecs;
250 Vector tps;
251
252 for (i = 0; i < tp.lexStates.length; i++)
253 {
254 if ((tps = (Vector)allTpsForState.get(tp.lexStates[i])) == null)
255 {
256 tmpLexStateName[maxLexStates++] = tp.lexStates[i];
257 allTpsForState.put(tp.lexStates[i], tps = new Vector());
258 }
259
260 tps.addElement(tp);
261 }
262
263 if (respecs == null || respecs.size() == 0)
264 continue;
265
266 RegularExpression re;
267 for (i = 0; i < respecs.size(); i++)
268 if (maxOrdinal <= (re = ((RegExprSpec)respecs.elementAt(i)).rexp).ordinal)
269 maxOrdinal = re.ordinal + 1;
270 }
271
272 kinds = new int[maxOrdinal];
273 toSkip = new long[maxOrdinal / 64 + 1];
274 toSpecial = new long[maxOrdinal / 64 + 1];
275 toMore = new long[maxOrdinal / 64 + 1];
276 toToken = new long[maxOrdinal / 64 + 1];
277 toToken[0] = 1L;
278 actions = new Action[maxOrdinal];
279 actions[0] = actForEof;
280 hasTokenActions = actForEof != null;
281 initStates = new Hashtable();
282 canMatchAnyChar = new int[maxLexStates];
283 canLoop = new boolean[maxLexStates];
284 stateHasActions = new boolean[maxLexStates];
285 lexStateName = new String [maxLexStates];
286 singlesToSkip = new NfaState[maxLexStates];
287 System.arraycopy(tmpLexStateName, 0, lexStateName, 0, maxLexStates);
288
289 for (i = 0; i < maxLexStates; i++)
290 canMatchAnyChar[i] = -1;
291
292 hasNfa = new boolean[maxLexStates];
293 mixed = new boolean[maxLexStates];
294 maxLongsReqd = new int[maxLexStates];
295 initMatch = new int[maxLexStates];
296 newLexState = new String [maxOrdinal];
297 newLexState[0] = nextStateForEof;
298 hasEmptyMatch = false;
299 lexStates = new int[maxOrdinal];
300 ignoreCase = new boolean[maxOrdinal];
301 rexprs = new RegularExpression[maxOrdinal];
302 RStringLiteral.allImages = new String [maxOrdinal];
303 canReachOnMore = new boolean[maxLexStates];
304 }
305
306 static int GetIndex(String name)
307 {
308 for (int i = 0; i < lexStateName.length; i++)
309 if (lexStateName[i] != null && lexStateName[i].equals(name))
310 return i;
311
312 throw new Error (); }
314
315 public static void AddCharToSkip(char c, int kind)
316 {
317 singlesToSkip[lexStateIndex].AddChar(c);
318 singlesToSkip[lexStateIndex].kind = kind;
319 }
320
321 public static void start()
322 {
323 if (!Options.getBuildTokenManager() ||
324 Options.getUserTokenManager() ||
325 JavaCCErrors.get_error_count() > 0)
326 return;
327
328 keepLineCol = Options.getKeepLineColumn();
329 Vector choices = new Vector();
330 Enumeration e;
331 TokenProduction tp;
332 int i, j;
333
334 staticString = (Options.getStatic() ? "static " : "");
335 tokMgrClassName = cu_name + "TokenManager";
336
337 PrintClassHead();
338 BuildLexStatesTable();
339
340 e = allTpsForState.keys();
341
342 boolean ignoring = false;
343
344 while (e.hasMoreElements())
345 {
346 NfaState.ReInit();
347 RStringLiteral.ReInit();
348
349 String key = (String )e.nextElement();
350
351 lexStateIndex = GetIndex(key);
352 lexStateSuffix = "_" + lexStateIndex;
353 Vector allTps = (Vector)allTpsForState.get(key);
354 initStates.put(key, initialState = new NfaState());
355 ignoring = false;
356
357 singlesToSkip[lexStateIndex] = new NfaState();
358 singlesToSkip[lexStateIndex].dummy = true;
359
360 if (key.equals("DEFAULT"))
361 defaultLexState = lexStateIndex;
362
363 for (i = 0; i < allTps.size(); i++)
364 {
365 tp = (TokenProduction)allTps.elementAt(i);
366 int kind = tp.kind;
367 boolean ignore = tp.ignoreCase;
368 Vector rexps = tp.respecs;
369
370 if (i == 0)
371 ignoring = ignore;
372
373 for (j = 0; j < rexps.size(); j++)
374 {
375 RegExprSpec respec = (RegExprSpec)rexps.elementAt(j);
376 curRE = respec.rexp;
377
378 rexprs[curKind = curRE.ordinal] = curRE;
379 lexStates[curRE.ordinal] = lexStateIndex;
380 ignoreCase[curRE.ordinal] = ignore;
381
382 if (curRE.private_rexp)
383 {
384 kinds[curRE.ordinal] = -1;
385 continue;
386 }
387
388 if (curRE instanceof RStringLiteral &&
389 !((RStringLiteral)curRE).image.equals(""))
390 {
391 ((RStringLiteral)curRE).GenerateDfa(ostr, curRE.ordinal);
392 if (i != 0 && !mixed[lexStateIndex] && ignoring != ignore)
393 mixed[lexStateIndex] = true;
394 }
395 else if (curRE.CanMatchAnyChar())
396 {
397 if (canMatchAnyChar[lexStateIndex] == -1 ||
398 canMatchAnyChar[lexStateIndex] > curRE.ordinal)
399 canMatchAnyChar[lexStateIndex] = curRE.ordinal;
400 }
401 else
402 {
403 Nfa temp;
404
405 if (curRE instanceof RChoice)
406 choices.addElement(curRE);
407
408 temp = curRE.GenerateNfa(ignore);
409 temp.end.isFinal = true;
410 temp.end.kind = curRE.ordinal;
411 initialState.AddMove(temp.start);
412 }
413
414 if (kinds.length < curRE.ordinal)
415 {
416 int[] tmp = new int[curRE.ordinal + 1];
417
418 System.arraycopy(kinds, 0, tmp, 0, kinds.length);
419 kinds = tmp;
420 }
421
423 kinds[curRE.ordinal] = kind;
424
425 if (respec.nextState != null &&
426 !respec.nextState.equals(lexStateName[lexStateIndex]))
427 newLexState[curRE.ordinal] = respec.nextState;
428
429 if (respec.act != null && respec.act.action_tokens != null &&
430 respec.act.action_tokens.size() > 0)
431 actions[curRE.ordinal] = respec.act;
432
433 switch(kind)
434 {
435 case TokenProduction.SPECIAL :
436 hasSkipActions |= (actions[curRE.ordinal] != null) ||
437 (newLexState[curRE.ordinal] != null);
438 hasSpecial = true;
439 toSpecial[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
440 toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
441 break;
442 case TokenProduction.SKIP :
443 hasSkipActions |= (actions[curRE.ordinal] != null);
444 hasSkip = true;
445 toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
446 break;
447 case TokenProduction.MORE :
448 hasMoreActions |= (actions[curRE.ordinal] != null);
449 hasMore = true;
450 toMore[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
451
452 if (newLexState[curRE.ordinal] != null)
453 canReachOnMore[GetIndex(newLexState[curRE.ordinal])] = true;
454 else
455 canReachOnMore[lexStateIndex] = true;
456
457 break;
458 case TokenProduction.TOKEN :
459 hasTokenActions |= (actions[curRE.ordinal] != null);
460 hasToken = true;
461 toToken[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
462 break;
463 }
464 }
465 }
466
467 NfaState.ComputeClosures();
469
470 for (i = 0; i < initialState.epsilonMoves.size(); i++)
471 ((NfaState)initialState.epsilonMoves.elementAt(i)).GenerateCode();
472
473 if (hasNfa[lexStateIndex] = (NfaState.generatedStates != 0))
474 {
475 initialState.GenerateCode();
476 initialState.GenerateInitMoves(ostr);
477 }
478
479 if (initialState.kind != Integer.MAX_VALUE && initialState.kind != 0)
480 {
481 if ((toSkip[initialState.kind / 64] & (1L << initialState.kind)) != 0L ||
482 (toSpecial[initialState.kind / 64] & (1L << initialState.kind)) != 0L)
483 hasSkipActions = true;
484 else if ((toMore[initialState.kind / 64] & (1L << initialState.kind)) != 0L)
485 hasMoreActions = true;
486 else
487 hasTokenActions = true;
488
489 if (initMatch[lexStateIndex] == 0 ||
490 initMatch[lexStateIndex] > initialState.kind)
491 {
492 initMatch[lexStateIndex] = initialState.kind;
493 hasEmptyMatch = true;
494 }
495 }
496 else if (initMatch[lexStateIndex] == 0)
497 initMatch[lexStateIndex] = Integer.MAX_VALUE;
498
499 RStringLiteral.FillSubString();
500
501 if (hasNfa[lexStateIndex] && !mixed[lexStateIndex])
502 RStringLiteral.GenerateNfaStartStates(ostr, initialState);
503
504 RStringLiteral.DumpDfaCode(ostr);
505
506 if (hasNfa[lexStateIndex])
507 NfaState.DumpMoveNfa(ostr);
508
509 if (stateSetSize < NfaState.generatedStates)
510 stateSetSize = NfaState.generatedStates;
511 }
512
513 for (i = 0; i < choices.size(); i++)
514 ((RChoice)choices.elementAt(i)).CheckUnmatchability();
515
516 NfaState.DumpStateSets(ostr);
517 CheckEmptyStringMatch();
518 NfaState.DumpNonAsciiMoveMethods(ostr);
519 RStringLiteral.DumpStrLiteralImages(ostr);
520 DumpStaticVarDeclarations();
521 DumpFillToken();
522 DumpGetNextToken();
523
524 if (Options.getDebugTokenManager())
525 {
526 NfaState.DumpStatesForKind(ostr);
527 DumpDebugMethods();
528 }
529
530 if (hasLoop)
531 {
532 ostr.println(staticString + "int[] jjemptyLineNo = new int[" + maxLexStates + "];");
533 ostr.println(staticString + "int[] jjemptyColNo = new int[" + maxLexStates + "];");
534 ostr.println(staticString + "boolean[] jjbeenHere = new boolean[" + maxLexStates + "];");
535 }
536
537 if (hasSkipActions)
538 DumpSkipActions();
539 if (hasMoreActions)
540 DumpMoreActions();
541 if (hasTokenActions)
542 DumpTokenActions();
543
544 ostr.println( "}");
545 ostr.close();
546 }
547
548 static void CheckEmptyStringMatch()
549 {
550 int i, j, k, len;
551 boolean[] seen = new boolean[maxLexStates];
552 boolean[] done = new boolean[maxLexStates];
553 String cycle;
554 String reList;
555
556 Outer:
557 for (i = 0; i < maxLexStates; i++)
558 {
559 if (done[i] || initMatch[i] == 0 || initMatch[i] == Integer.MAX_VALUE ||
560 canMatchAnyChar[i] != -1)
561 continue;
562
563 done[i] = true;
564 len = 0;
565 cycle = "";
566 reList = "";
567
568 for (k = 0; k < maxLexStates; k++)
569 seen[k] = false;
570
571 j = i;
572 seen[i] = true;
573 cycle += lexStateName[j] + "-->";
574 while (newLexState[initMatch[j]] != null)
575 {
576 cycle += newLexState[initMatch[j]];
577 if (seen[j = GetIndex(newLexState[initMatch[j]])])
578 break;
579
580 cycle += "-->";
581 done[j] = true;
582 seen[j] = true;
583 if (initMatch[j] == 0 || initMatch[j] == Integer.MAX_VALUE ||
584 canMatchAnyChar[j] != -1)
585 continue Outer;
586 if (len != 0)
587 reList += "; ";
588 reList += "line " + rexprs[initMatch[j]].line + ", column " +
589 rexprs[initMatch[j]].column;
590 len++;
591 }
592
593 if (newLexState[initMatch[j]] == null)
594 cycle += lexStateName[lexStates[initMatch[j]]];
595
596 for (k = 0; k < maxLexStates; k++)
597 canLoop[k] |= seen[k];
598
599 hasLoop = true;
600 if (len == 0)
601 JavaCCErrors.warning(rexprs[initMatch[i]],
602 "Regular expression" + ((rexprs[initMatch[i]].label.equals(""))
603 ? "" : (" for " + rexprs[initMatch[i]].label)) +
604 " can be matched by the empty string (\"\") in lexical state " +
605 lexStateName[i] + ". This can result in an endless loop of " +
606 "empty string matches.");
607 else
608 {
609 JavaCCErrors.warning(rexprs[initMatch[i]],
610 "Regular expression" + ((rexprs[initMatch[i]].label.equals(""))
611 ? "" : (" for " + rexprs[initMatch[i]].label)) +
612 " can be matched by the empty string (\"\") in lexical state " +
613 lexStateName[i] + ". This regular expression along with the " +
614 "regular expressions at " + reList + " forms the cycle \n " +
615 cycle + "\ncontaining regular expressions with empty matches." +
616 " This can result in an endless loop of empty string matches.");
617 }
618 }
619 }
620
621 static void PrintArrayInitializer(int noElems)
622 {
623 ostr.print("{");
624 for (int i = 0; i < noElems; i++)
625 {
626 if (i % 25 == 0)
627 ostr.print("\n ");
628 ostr.print("0, ");
629 }
630 ostr.println("\n};");
631 }
632
633 static void DumpStaticVarDeclarations()
634 {
635 int i;
636 String charStreamName;
637
638 ostr.println("public static final String[] lexStateNames = {");
639 for (i = 0; i < maxLexStates; i++)
640 ostr.println(" \"" + lexStateName[i] + "\", ");
641 ostr.println("};");
642
643 if (maxLexStates > 1)
644 {
645 ostr.print("public static final int[] jjnewLexState = {");
646
647 for (i = 0; i < maxOrdinal; i++)
648 {
649 if (i % 25 == 0)
650 ostr.print("\n ");
651
652 if (newLexState[i] == null)
653 ostr.print("-1, ");
654 else
655 ostr.print(GetIndex(newLexState[i]) + ", ");
656 }
657 ostr.println("\n};");
658 }
659
660 if (hasSkip || hasMore || hasSpecial)
661 {
662 ostr.print("static final long[] jjtoToken = {");
664 for (i = 0; i < maxOrdinal / 64 + 1; i++)
665 {
666 if (i % 4 == 0)
667 ostr.print("\n ");
668 ostr.print("0x" + Long.toHexString(toToken[i]) + "L, ");
669 }
670 ostr.println("\n};");
671 }
672
673 if (hasSkip || hasSpecial)
674 {
675 ostr.print("static final long[] jjtoSkip = {");
677 for (i = 0; i < maxOrdinal / 64 + 1; i++)
678 {
679 if (i % 4 == 0)
680 ostr.print("\n ");
681 ostr.print("0x" + Long.toHexString(toSkip[i]) + "L, ");
682 }
683 ostr.println("\n};");
684 }
685
686 if (hasSpecial)
687 {
688 ostr.print("static final long[] jjtoSpecial = {");
690 for (i = 0; i < maxOrdinal / 64 + 1; i++)
691 {
692 if (i % 4 == 0)
693 ostr.print("\n ");
694 ostr.print("0x" + Long.toHexString(toSpecial[i]) + "L, ");
695 }
696 ostr.println("\n};");
697 }
698
699 if (hasMore)
700 {
701 ostr.print("static final long[] jjtoMore = {");
703 for (i = 0; i < maxOrdinal / 64 + 1; i++)
704 {
705 if (i % 4 == 0)
706 ostr.print("\n ");
707 ostr.print("0x" + Long.toHexString(toMore[i]) + "L, ");
708 }
709 ostr.println("\n};");
710 }
711
712 if (Options.getUserCharStream())
713 charStreamName = "CharStream";
714 else
715 {
716 if (Options.getJavaUnicodeEscape())
717 charStreamName = "JavaCharStream";
718 else
719 charStreamName = "SimpleCharStream";
720 }
721
722 ostr.println(staticString + "protected " + charStreamName + " input_stream;");
723
724 ostr.println(staticString + "private final int[] jjrounds = " +
725 "new int[" + stateSetSize + "];");
726 ostr.println(staticString + "private final int[] jjstateSet = " +
727 "new int[" + (2 * stateSetSize) + "];");
728
729 if (hasMoreActions || hasSkipActions || hasTokenActions)
730 {
731 ostr.println(staticString + "StringBuffer image;");
732 ostr.println(staticString + "int jjimageLen;");
733 ostr.println(staticString + "int lengthOfMatch;");
734 }
735
736 ostr.println(staticString + "protected char curChar;");
737
738 if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
739 ostr.println("public " + tokMgrClassName + "(" + cu_name + " parserArg, " + charStreamName + " stream){");
740 ostr.println(" parser = parserArg;");
741 } else {
742 ostr.println("public " + tokMgrClassName + "(" + charStreamName + " stream){");
743 }
744
745 if (Options.getStatic() && !Options.getUserCharStream())
746 {
747 ostr.println(" if (input_stream != null)");
748 ostr.println(" throw new TokenMgrError(\"ERROR: Second call to constructor of static lexer. You must use ReInit() to initialize the static variables.\", TokenMgrError.STATIC_LEXER_ERROR);");
749 }
750 else if (!Options.getUserCharStream())
751 {
752 if (Options.getJavaUnicodeEscape())
753 ostr.println(" if (JavaCharStream.staticFlag)");
754 else
755 ostr.println(" if (SimpleCharStream.staticFlag)");
756
757 ostr.println(" throw new Error(\"ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.\");");
758 }
759
760 ostr.println(" input_stream = stream;");
761
762 ostr.println("}");
763
764 if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
765 ostr.println("public " + tokMgrClassName + "(" + cu_name + " parserArg, " + charStreamName + " stream, int lexState){");
766 ostr.println(" this(parserArg, stream);");
767 } else {
768 ostr.println("public " + tokMgrClassName + "(" + charStreamName + " stream, int lexState){");
769 ostr.println(" this(stream);");
770 }
771 ostr.println(" SwitchTo(lexState);");
772 ostr.println("}");
773
774 ostr.println(staticString + "public void ReInit(" + charStreamName + " stream)");
776 ostr.println("{");
777 ostr.println(" jjmatchedPos = jjnewStateCnt = 0;");
778 ostr.println(" curLexState = defaultLexState;");
779 ostr.println(" input_stream = stream;");
780 ostr.println(" ReInitRounds();");
781 ostr.println("}");
782
783 ostr.println(staticString + "private final void ReInitRounds()");
785 ostr.println("{");
786 ostr.println(" int i;");
787 ostr.println(" jjround = 0x" + Integer.toHexString(Integer.MIN_VALUE + 1)+ ";");
788 ostr.println(" for (i = " + stateSetSize + "; i-- > 0;)");
789 ostr.println(" jjrounds[i] = 0x" + Integer.toHexString(Integer.MIN_VALUE) + ";");
790 ostr.println("}");
791
792 ostr.println(staticString + "public void ReInit(" + charStreamName + " stream, int lexState)");
794 ostr.println("{");
795 ostr.println(" ReInit(stream);");
796 ostr.println(" SwitchTo(lexState);");
797 ostr.println("}");
798
799 ostr.println(staticString + "public void SwitchTo(int lexState)");
800 ostr.println("{");
801 ostr.println(" if (lexState >= " + lexStateName.length + " || lexState < 0)");
802 ostr.println(" throw new TokenMgrError(\"Error: Ignoring invalid lexical state : \"" +
803 " + lexState + \". State unchanged.\", TokenMgrError.INVALID_LEXICAL_STATE);");
804 ostr.println(" else");
805 ostr.println(" curLexState = lexState;");
806 ostr.println("}");
807
808 ostr.println("");
809 }
810
811 static char MaxChar(long l)
813 {
814 for (int i = 64; i-- > 0; )
815 if ((l & (1L << i)) != 0L)
816 return (char)i;
817
818 return 0xffff;
819 }
820
821 static void DumpFillToken()
822 {
823 ostr.println(staticString + "protected Token jjFillToken()");
824 ostr.println("{");
825 ostr.println(" Token t = Token.newToken(jjmatchedKind);");
826 ostr.println(" t.kind = jjmatchedKind;");
827
828 if (hasEmptyMatch)
829 {
830 ostr.println(" if (jjmatchedPos < 0)");
831 ostr.println(" {");
832 ostr.println(" if (image == null)");
833 ostr.println(" t.image = \"\";");
834 ostr.println(" else");
835 ostr.println(" t.image = image.toString();");
836
837 if (keepLineCol)
838 {
839 ostr.println(" t.beginLine = t.endLine = input_stream.getBeginLine();");
840 ostr.println(" t.beginColumn = t.endColumn = input_stream.getBeginColumn();");
841 }
842
843 ostr.println(" }");
844 ostr.println(" else");
845 ostr.println(" {");
846 ostr.println(" String im = jjstrLiteralImages[jjmatchedKind];");
847 ostr.println(" t.image = (im == null) ? input_stream.GetImage() : im;");
848
849 if (keepLineCol)
850 {
851 ostr.println(" t.beginLine = input_stream.getBeginLine();");
852 ostr.println(" t.beginColumn = input_stream.getBeginColumn();");
853 ostr.println(" t.endLine = input_stream.getEndLine();");
854 ostr.println(" t.endColumn = input_stream.getEndColumn();");
855 }
856
857 ostr.println(" }");
858 }
859 else
860 {
861 ostr.println(" String im = jjstrLiteralImages[jjmatchedKind];");
862 ostr.println(" t.image = (im == null) ? input_stream.GetImage() : im;");
863
864 if (keepLineCol)
865 {
866 ostr.println(" t.beginLine = input_stream.getBeginLine();");
867 ostr.println(" t.beginColumn = input_stream.getBeginColumn();");
868 ostr.println(" t.endLine = input_stream.getEndLine();");
869 ostr.println(" t.endColumn = input_stream.getEndColumn();");
870 }
871
872 }
873
874 ostr.println(" return t;");
875 ostr.println("}");
876 }
877
878 static void DumpGetNextToken()
879 {
880 int i;
881
882 ostr.println("");
883 ostr.println(staticString + "int curLexState = " + defaultLexState + ";");
884 ostr.println(staticString + "int defaultLexState = " + defaultLexState + ";");
885 ostr.println(staticString + "int jjnewStateCnt;");
886 ostr.println(staticString + "int jjround;");
887 ostr.println(staticString + "int jjmatchedPos;");
888 ostr.println(staticString + "int jjmatchedKind;");
889 ostr.println("");
890 ostr.println("public " + staticString + "Token getNextToken()" +
891 " ");
892 ostr.println("{");
893 ostr.println(" int kind;");
894 ostr.println(" Token specialToken = null;");
895 ostr.println(" Token matchedToken;");
896 ostr.println(" int curPos = 0;");
897 ostr.println("");
898 ostr.println(" EOFLoop :\n for (;;)");
899 ostr.println(" { ");
900 ostr.println(" try ");
901 ostr.println(" { ");
902 ostr.println(" curChar = input_stream.BeginToken();");
903 ostr.println(" } ");
904 ostr.println(" catch(java.io.IOException e)");
905 ostr.println(" { ");
906
907 if (Options.getDebugTokenManager())
908 ostr.println(" debugStream.println(\"Returning the <EOF> token.\");");
909
910 ostr.println(" jjmatchedKind = 0;");
911 ostr.println(" matchedToken = jjFillToken();");
912
913 if (hasSpecial)
914 ostr.println(" matchedToken.specialToken = specialToken;");
915
916 if (nextStateForEof != null || actForEof != null)
917 ostr.println(" TokenLexicalActions(matchedToken);");
918
919 if (Options.getCommonTokenAction())
920 ostr.println(" CommonTokenAction(matchedToken);");
921
922 ostr.println(" return matchedToken;");
923 ostr.println(" }");
924
925 if (hasMoreActions || hasSkipActions || hasTokenActions)
926 {
927 ostr.println(" image = null;");
928 ostr.println(" jjimageLen = 0;");
929 }
930
931 ostr.println("");
932
933 String prefix = "";
934 if (hasMore)
935 {
936 ostr.println(" for (;;)");
937 ostr.println(" {");
938 prefix = " ";
939 }
940
941 String endSwitch = "";
942 String caseStr = "";
943 if (maxLexStates > 1)
945 {
946 ostr.println(prefix + " switch(curLexState)");
947 ostr.println(prefix + " {");
948 endSwitch = prefix + " }";
949 caseStr = prefix + " case ";
950 prefix += " ";
951 }
952
953 prefix += " ";
954 for(i = 0; i < maxLexStates; i++)
955 {
956 if (maxLexStates > 1)
957 ostr.println(caseStr + i + ":");
958
959 if (singlesToSkip[i].HasTransitions())
960 {
961 ostr.println(prefix + "try { input_stream.backup(0);");
963 if (singlesToSkip[i].asciiMoves[0] != 0L &&
964 singlesToSkip[i].asciiMoves[1] != 0L)
965 {
966 ostr.println(prefix + " while ((curChar < 64" + " && (0x" +
967 Long.toHexString(singlesToSkip[i].asciiMoves[0]) +
968 "L & (1L << curChar)) != 0L) || \n" +
969 prefix + " (curChar >> 6) == 1" +
970 " && (0x" +
971 Long.toHexString(singlesToSkip[i].asciiMoves[1]) +
972 "L & (1L << (curChar & 077))) != 0L)");
973 }
974 else if (singlesToSkip[i].asciiMoves[1] == 0L)
975 {
976 ostr.println(prefix + " while (curChar <= " +
977 (int)MaxChar(singlesToSkip[i].asciiMoves[0]) + " && (0x" +
978 Long.toHexString(singlesToSkip[i].asciiMoves[0]) +
979 "L & (1L << curChar)) != 0L)");
980 }
981 else if (singlesToSkip[i].asciiMoves[0] == 0L)
982 {
983 ostr.println(prefix + " while (curChar > 63 && curChar <= " +
984 ((int)MaxChar(singlesToSkip[i].asciiMoves[1]) + 64) +
985 " && (0x" +
986 Long.toHexString(singlesToSkip[i].asciiMoves[1]) +
987 "L & (1L << (curChar & 077))) != 0L)");
988 }
989
990 if (Options.getDebugTokenManager())
991 {
992 ostr.println(prefix + "{");
993 ostr.println(" debugStream.println(" + (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") + "\"Skipping character : \" + " +
994 "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \")\");");
995 }
996 ostr.println(prefix + " curChar = input_stream.BeginToken();");
997
998 if (Options.getDebugTokenManager())
999 ostr.println(prefix + "}");
1000
1001 ostr.println(prefix + "}");
1002 ostr.println(prefix + "catch (java.io.IOException e1) { continue EOFLoop; }");
1003 }
1004
1005 if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
1006 {
1007 if (Options.getDebugTokenManager())
1008 ostr.println(" debugStream.println(\" Matched the empty string as \" + tokenImage[" +
1009 initMatch[i] + "] + \" token.\");");
1010
1011 ostr.println(prefix + "jjmatchedKind = " + initMatch[i] + ";");
1012 ostr.println(prefix + "jjmatchedPos = -1;");
1013 ostr.println(prefix + "curPos = 0;");
1014 }
1015 else
1016 {
1017 ostr.println(prefix + "jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
1018 ostr.println(prefix + "jjmatchedPos = 0;");
1019 }
1020
1021 if (Options.getDebugTokenManager())
1022 ostr.println(" debugStream.println(" + (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") + "\"Current character : \" + " +
1023 "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") at line \" + input_stream.getLine() + \" column \" + input_stream.getColumn());");
1024
1025 ostr.println(prefix + "curPos = jjMoveStringLiteralDfa0_" + i + "();");
1026
1027 if (canMatchAnyChar[i] != -1)
1028 {
1029 if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
1030 ostr.println(prefix + "if (jjmatchedPos < 0 || (jjmatchedPos == 0 && jjmatchedKind > " +
1031 canMatchAnyChar[i] + "))");
1032 else
1033 ostr.println(prefix + "if (jjmatchedPos == 0 && jjmatchedKind > " +
1034 canMatchAnyChar[i] + ")");
1035 ostr.println(prefix + "{");
1036
1037 if (Options.getDebugTokenManager())
1038 ostr.println(" debugStream.println(\" Current character matched as a \" + tokenImage[" +
1039 canMatchAnyChar[i] + "] + \" token.\");");
1040 ostr.println(prefix + " jjmatchedKind = " + canMatchAnyChar[i] + ";");
1041
1042 if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
1043 ostr.println(prefix + " jjmatchedPos = 0;");
1044
1045 ostr.println(prefix + "}");
1046 }
1047
1048 if (maxLexStates > 1)
1049 ostr.println(prefix + "break;");
1050 }
1051
1052 if (maxLexStates > 1)
1053 ostr.println(endSwitch);
1054 else if (maxLexStates == 0)
1055 ostr.println(" jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
1056
1057 if (maxLexStates > 1)
1058 prefix = " ";
1059 else
1060 prefix = "";
1061
1062 if (maxLexStates > 0)
1063 {
1064 ostr.println(prefix + " if (jjmatchedKind != 0x" + Integer.toHexString(Integer.MAX_VALUE) + ")");
1065 ostr.println(prefix + " {");
1066 ostr.println(prefix + " if (jjmatchedPos + 1 < curPos)");
1067
1068 if (Options.getDebugTokenManager())
1069 {
1070 ostr.println(prefix + " {");
1071 ostr.println(prefix + " debugStream.println(\" Putting back \" + (curPos - jjmatchedPos - 1) + \" characters into the input stream.\");");
1072 }
1073
1074 ostr.println(prefix + " input_stream.backup(curPos - jjmatchedPos - 1);");
1075
1076 if (Options.getDebugTokenManager())
1077 ostr.println(prefix + " }");
1078
1079 if (Options.getDebugTokenManager())
1080 {
1081 if (Options.getJavaUnicodeEscape() ||
1082 Options.getUserCharStream())
1083 ostr.println(" debugStream.println(\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH (\" + TokenMgrError.addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + \") ******\\n\");");
1084 else
1085 ostr.println(" debugStream.println(\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH (\" + TokenMgrError.addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + \") ******\\n\");");
1086 }
1087
1088 if (hasSkip || hasMore || hasSpecial)
1089 {
1090 ostr.println(prefix + " if ((jjtoToken[jjmatchedKind >> 6] & " +
1091 "(1L << (jjmatchedKind & 077))) != 0L)");
1092 ostr.println(prefix + " {");
1093 }
1094
1095 ostr.println(prefix + " matchedToken = jjFillToken();");
1096
1097 if (hasSpecial)
1098 ostr.println(prefix + " matchedToken.specialToken = specialToken;");
1099
1100 if (hasTokenActions)
1101 ostr.println(prefix + " TokenLexicalActions(matchedToken);");
1102
1103 if (maxLexStates > 1)
1104 {
1105 ostr.println(" if (jjnewLexState[jjmatchedKind] != -1)");
1106 ostr.println(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
1107 }
1108
1109 if (Options.getCommonTokenAction())
1110 ostr.println(prefix + " CommonTokenAction(matchedToken);");
1111
1112 ostr.println(prefix + " return matchedToken;");
1113
1114 if (hasSkip || hasMore || hasSpecial)
1115 {
1116 ostr.println(prefix + " }");
1117
1118 if (hasSkip || hasSpecial)
1119 {
1120 if (hasMore)
1121 {
1122 ostr.println(prefix + " else if ((jjtoSkip[jjmatchedKind >> 6] & " +
1123 "(1L << (jjmatchedKind & 077))) != 0L)");
1124 }
1125 else
1126 ostr.println(prefix + " else");
1127
1128 ostr.println(prefix + " {");
1129
1130 if (hasSpecial)
1131 {
1132 ostr.println(prefix + " if ((jjtoSpecial[jjmatchedKind >> 6] & " +
1133 "(1L << (jjmatchedKind & 077))) != 0L)");
1134 ostr.println(prefix + " {");
1135
1136 ostr.println(prefix + " matchedToken = jjFillToken();");
1137
1138 ostr.println(prefix + " if (specialToken == null)");
1139 ostr.println(prefix + " specialToken = matchedToken;");
1140 ostr.println(prefix + " else");
1141 ostr.println(prefix + " {");
1142 ostr.println(prefix + " matchedToken.specialToken = specialToken;");
1143 ostr.println(prefix + " specialToken = (specialToken.next = matchedToken);");
1144 ostr.println(prefix + " }");
1145
1146 if (hasSkipActions)
1147 ostr.println(prefix + " SkipLexicalActions(matchedToken);");
1148
1149 ostr.println(prefix + " }");
1150
1151 if (hasSkipActions)
1152 {
1153 ostr.println(prefix + " else ");
1154 ostr.println(prefix + " SkipLexicalActions(null);");
1155 }
1156 }
1157 else if (hasSkipActions)
1158 ostr.println(prefix + " SkipLexicalActions(null);");
1159
1160 if (maxLexStates > 1)
1161 {
1162 ostr.println(" if (jjnewLexState[jjmatchedKind] != -1)");
1163 ostr.println(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
1164 }
1165
1166 ostr.println(prefix + " continue EOFLoop;");
1167 ostr.println(prefix + " }");
1168 }
1169
1170 if (hasMore)
1171 {
1172 if (hasMoreActions)
1173 ostr.println(prefix + " MoreLexicalActions();");
1174 else if (hasSkipActions || hasTokenActions)
1175 ostr.println(prefix + " jjimageLen += jjmatchedPos + 1;");
1176
1177 if (maxLexStates > 1)
1178 {
1179 ostr.println(" if (jjnewLexState[jjmatchedKind] != -1)");
1180 ostr.println(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
1181 }
1182 ostr.println(prefix + " curPos = 0;");
1183 ostr.println(prefix + " jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
1184
1185 ostr.println(prefix + " try {");
1186 ostr.println(prefix + " curChar = input_stream.readChar();");
1187
1188 if (Options.getDebugTokenManager())
1189 ostr.println(" debugStream.println(" + (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") + "\"Current character : \" + " +
1190 "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") at line \" + input_stream.getLine() + \" column \" + input_stream.getColumn());");
1191 ostr.println(prefix + " continue;");
1192 ostr.println(prefix + " }");
1193 ostr.println(prefix + " catch (java.io.IOException e1) { }");
1194 }
1195 }
1196
1197 ostr.println(prefix + " }");
1198 ostr.println(prefix + " int error_line = input_stream.getEndLine();");
1199 ostr.println(prefix + " int error_column = input_stream.getEndColumn();");
1200 ostr.println(prefix + " String error_after = null;");
1201 ostr.println(prefix + " boolean EOFSeen = false;");
1202 ostr.println(prefix + " try { input_stream.readChar(); input_stream.backup(1); }");
1203 ostr.println(prefix + " catch (java.io.IOException e1) {");
1204 ostr.println(prefix + " EOFSeen = true;");
1205 ostr.println(prefix + " error_after = curPos <= 1 ? \"\" : input_stream.GetImage();");
1206 ostr.println(prefix + " if (curChar == '\\n' || curChar == '\\r') {");
1207 ostr.println(prefix + " error_line++;");
1208 ostr.println(prefix + " error_column = 0;");
1209 ostr.println(prefix + " }");
1210 ostr.println(prefix + " else");
1211 ostr.println(prefix + " error_column++;");
1212 ostr.println(prefix + " }");
1213 ostr.println(prefix + " if (!EOFSeen) {");
1214 ostr.println(prefix + " input_stream.backup(1);");
1215 ostr.println(prefix + " error_after = curPos <= 1 ? \"\" : input_stream.GetImage();");
1216 ostr.println(prefix + " }");
1217 ostr.println(prefix + " throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);");
1218 }
1219
1220 if (hasMore)
1221 ostr.println(prefix + " }");
1222
1223 ostr.println(" }");
1224 ostr.println("}");
1225 ostr.println("");
1226 }
1227
1228 public static void DumpSkipActions()
1229 {
1230 Action act;
1231
1232 ostr.println(staticString + "void SkipLexicalActions(Token matchedToken)");
1233 ostr.println("{");
1234 ostr.println(" switch(jjmatchedKind)");
1235 ostr.println(" {");
1236
1237 Outer:
1238 for (int i = 0; i < maxOrdinal; i++)
1239 {
1240 if ((toSkip[i / 64] & (1L << (i % 64))) == 0L)
1241 continue;
1242
1243 for (;;)
1244 {
1245 if (((act = (Action)actions[i]) == null ||
1246 act.action_tokens == null ||
1247 act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
1248 continue Outer;
1249
1250 ostr.println(" case " + i + " :");
1251
1252 if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
1253 {
1254 ostr.println(" if (jjmatchedPos == -1)");
1255 ostr.println(" {");
1256 ostr.println(" if (jjbeenHere[" + lexStates[i] + "] &&");
1257 ostr.println(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
1258 ostr.println(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
1259 ostr.println(" throw new TokenMgrError((\"Error: Bailing out of infinite loop caused by repeated empty string matches at line \" + input_stream.getBeginLine() + \", column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
1260 ostr.println(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
1261 ostr.println(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
1262 ostr.println(" jjbeenHere[" + lexStates[i] + "] = true;");
1263 ostr.println(" }");
1264 }
1265
1266 if ((act = (Action)actions[i]) == null ||
1267 act.action_tokens.size() == 0)
1268 break;
1269
1270 ostr.println(" if (image == null)");
1271 ostr.println(" image = new StringBuffer();");
1272
1273 ostr.print(" image.append");
1274 if (RStringLiteral.allImages[i] != null)
1275 ostr.println("(jjstrLiteralImages[" + i + "]);");
1276 else
1277 if (Options.getJavaUnicodeEscape() ||
1278 Options.getUserCharStream())
1279 ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
1280 else
1281 ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
1282
1283 printTokenSetup((Token)act.action_tokens.elementAt(0));
1284 ccol = 1;
1285
1286 for (int j = 0; j < act.action_tokens.size(); j++)
1287 printToken((Token)act.action_tokens.elementAt(j), ostr);
1288 ostr.println("");
1289
1290 break;
1291 }
1292
1293 ostr.println(" break;");
1294 }
1295
1296 ostr.println(" default :");
1297 ostr.println(" break;");
1298 ostr.println(" }");
1299 ostr.println("}");
1300 }
1301
1302 public static void DumpMoreActions()
1303 {
1304 Action act;
1305
1306 ostr.println(staticString + "void MoreLexicalActions()");
1307 ostr.println("{");
1308 ostr.println(" jjimageLen += (lengthOfMatch = jjmatchedPos + 1);");
1309 ostr.println(" switch(jjmatchedKind)");
1310 ostr.println(" {");
1311
1312 Outer:
1313 for (int i = 0; i < maxOrdinal; i++)
1314 {
1315 if ((toMore[i / 64] & (1L << (i % 64))) == 0L)
1316 continue;
1317
1318 for (;;)
1319 {
1320 if (((act = (Action)actions[i]) == null ||
1321 act.action_tokens == null ||
1322 act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
1323 continue Outer;
1324
1325 ostr.println(" case " + i + " :");
1326
1327 if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
1328 {
1329 ostr.println(" if (jjmatchedPos == -1)");
1330 ostr.println(" {");
1331 ostr.println(" if (jjbeenHere[" + lexStates[i] + "] &&");
1332 ostr.println(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
1333 ostr.println(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
1334 ostr.println(" throw new TokenMgrError((\"Error: Bailing out of infinite loop caused by repeated empty string matches at line \" + input_stream.getBeginLine() + \", column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
1335 ostr.println(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
1336 ostr.println(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
1337 ostr.println(" jjbeenHere[" + lexStates[i] + "] = true;");
1338 ostr.println(" }");
1339 }
1340
1341 if ((act = (Action)actions[i]) == null ||
1342 act.action_tokens.size() == 0)
1343 {
1344 break;
1345 }
1346
1347 ostr.println(" if (image == null)");
1348 ostr.println(" image = new StringBuffer();");
1349 ostr.print(" image.append");
1350
1351 if (RStringLiteral.allImages[i] != null)
1352 ostr.println("(jjstrLiteralImages[" + i + "]);");
1353 else
1354 if (Options.getJavaUnicodeEscape() ||
1355 Options.getUserCharStream())
1356 ostr.println("(input_stream.GetSuffix(jjimageLen));");
1357 else
1358 ostr.println("(input_stream.GetSuffix(jjimageLen));");
1359
1360 ostr.println(" jjimageLen = 0;");
1361 printTokenSetup((Token)act.action_tokens.elementAt(0));
1362 ccol = 1;
1363
1364 for (int j = 0; j < act.action_tokens.size(); j++)
1365 printToken((Token)act.action_tokens.elementAt(j), ostr);
1366 ostr.println("");
1367
1368 break;
1369 }
1370
1371 ostr.println(" break;");
1372 }
1373
1374 ostr.println(" default : ");
1375 ostr.println(" break;");
1376
1377 ostr.println(" }");
1378 ostr.println("}");
1379 }
1380
1381 public static void DumpTokenActions()
1382 {
1383 Action act;
1384 int i;
1385
1386 ostr.println(staticString + "void TokenLexicalActions(Token matchedToken)");
1387 ostr.println("{");
1388 ostr.println(" switch(jjmatchedKind)");
1389 ostr.println(" {");
1390
1391 Outer:
1392 for (i = 0; i < maxOrdinal; i++)
1393 {
1394 if ((toToken[i / 64] & (1L << (i % 64))) == 0L)
1395 continue;
1396
1397 for (;;)
1398 {
1399 if (((act = (Action)actions[i]) == null ||
1400 act.action_tokens == null ||
1401 act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
1402 continue Outer;
1403
1404 ostr.println(" case " + i + " :");
1405
1406 if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
1407 {
1408 ostr.println(" if (jjmatchedPos == -1)");
1409 ostr.println(" {");
1410 ostr.println(" if (jjbeenHere[" + lexStates[i] + "] &&");
1411 ostr.println(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
1412 ostr.println(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
1413 ostr.println(" throw new TokenMgrError((\"Error: Bailing out of infinite loop caused by repeated empty string matches at line \" + input_stream.getBeginLine() + \", column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
1414 ostr.println(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
1415 ostr.println(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
1416 ostr.println(" jjbeenHere[" + lexStates[i] + "] = true;");
1417 ostr.println(" }");
1418 }
1419
1420 if ((act = (Action)actions[i]) == null ||
1421 act.action_tokens.size() == 0)
1422 break;
1423
1424 if (i == 0)
1425 {
1426 ostr.println(" image = null;"); }
1428 else
1429 {
1430 ostr.println(" if (image == null)");
1431 ostr.println(" image = new StringBuffer();");
1432 ostr.print(" image.append");
1433
1434 if (RStringLiteral.allImages[i] != null)
1435 ostr.println("(jjstrLiteralImages[" + i + "]);");
1436 else
1437 if (Options.getJavaUnicodeEscape() ||
1438 Options.getUserCharStream())
1439 ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
1440 else
1441 ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
1442
1443 }
1444 printTokenSetup((Token)act.action_tokens.elementAt(0));
1445 ccol = 1;
1446
1447 for (int j = 0; j < act.action_tokens.size(); j++)
1448 printToken((Token)act.action_tokens.elementAt(j), ostr);
1449 ostr.println("");
1450
1451 break;
1452 }
1453
1454 ostr.println(" break;");
1455 }
1456
1457 ostr.println(" default : ");
1458 ostr.println(" break;");
1459 ostr.println(" }");
1460 ostr.println("}");
1461 }
1462
1463 public static void reInit()
1464 {
1465 ostr = null;
1466 staticString = null;
1467 tokMgrClassName = null;
1468 allTpsForState = new Hashtable();
1469 lexStateIndex = 0;
1470 kinds = null;
1471 maxOrdinal = 1;
1472 lexStateSuffix = null;
1473 newLexState = null;
1474 lexStates = null;
1475 ignoreCase = null;
1476 actions = null;
1477 initStates = new Hashtable();
1478 stateSetSize = 0;
1479 maxLexStates = 0;
1480 lexStateName = null;
1481 singlesToSkip = null;
1482 toSkip = null;
1483 toSpecial = null;
1484 toMore = null;
1485 toToken = null;
1486 defaultLexState = 0;
1487 rexprs = null;
1488 maxLongsReqd = null;
1489 initMatch = null;
1490 canMatchAnyChar = null;
1491 hasEmptyMatch = false;
1492 canLoop = null;
1493 stateHasActions = null;
1494 hasLoop = false;
1495 canReachOnMore = null;
1496 hasNfa = null;
1497 mixed = null;
1498 initialState = null;
1499 curKind = 0;
1500 hasSkipActions = false;
1501 hasMoreActions = false;
1502 hasTokenActions = false;
1503 hasSpecial = false;
1504 hasSkip = false;
1505 hasMore = false;
1506 hasToken = false;
1507 curRE = null;
1508 }
1509
1510}
1511 |
Popular Tags |
Java API By Example, From Geeks To Geeks.