KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > javacc > parser > LexGen


1 /*
2  * Copyright © 2002 Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
3  * California 95054, U.S.A. All rights reserved. Sun Microsystems, Inc. has
4  * intellectual property rights relating to technology embodied in the product
5  * that is described in this document. In particular, and without limitation,
6  * these intellectual property rights may include one or more of the U.S.
7  * patents listed at http://www.sun.com/patents and one or more additional
8  * patents or pending patent applications in the U.S. and in other countries.
9  * U.S. Government Rights - Commercial software. Government users are subject
10  * to the Sun Microsystems, Inc. standard license agreement and applicable
11  * provisions of the FAR and its supplements. Use is subject to license terms.
12  * Sun, Sun Microsystems, the Sun logo and Java are trademarks or registered
13  * trademarks of Sun Microsystems, Inc. in the U.S. and other countries. This
14  * product is covered and controlled by U.S. Export Control laws and may be
15  * subject to the export or import laws in other countries. Nuclear, missile,
16  * chemical biological weapons or nuclear maritime end uses or end users,
17  * whether direct or indirect, are strictly prohibited. Export or reexport
18  * to countries subject to U.S. embargo or to entities identified on U.S.
19  * export exclusion lists, including, but not limited to, the denied persons
20  * and specially designated nationals lists is strictly prohibited.
21  */

22
23 package org.javacc.parser;
24
25 import java.util.*;
26 import java.io.*;
27
28 public class LexGen
29     extends JavaCCGlobals
30     implements JavaCCParserConstants
31 {
32   static private java.io.PrintWriter JavaDoc ostr;
33   static private String JavaDoc staticString;
34   static private String JavaDoc tokMgrClassName;
35
36   // Hashtable of vectors
37
static Hashtable allTpsForState = new Hashtable();
38   public static int lexStateIndex = 0;
39   static int[] kinds;
40   public static int maxOrdinal = 1;
41   public static String JavaDoc lexStateSuffix;
42   static String JavaDoc[] newLexState;
43   public static int[] lexStates;
44   public static boolean[] ignoreCase;
45   public static Action[] actions;
46   public static Hashtable initStates = new Hashtable();
47   public static int stateSetSize;
48   public static int maxLexStates;
49   public static String JavaDoc[] lexStateName;
50   static NfaState[] singlesToSkip;
51   public static long[] toSkip;
52   public static long[] toSpecial;
53   public static long[] toMore;
54   public static long[] toToken;
55   public static int defaultLexState;
56   public static RegularExpression[] rexprs;
57   public static int[] maxLongsReqd;
58   public static int[] initMatch;
59   public static int[] canMatchAnyChar;
60   public static boolean hasEmptyMatch;
61   public static boolean[] canLoop;
62   public static boolean[] stateHasActions;
63   public static boolean hasLoop = false;
64   public static boolean[] canReachOnMore;
65   public static boolean[] hasNfa;
66   public static boolean[] mixed;
67   public static NfaState initialState;
68   public static int curKind;
69   static boolean hasSkipActions = false;
70   static boolean hasMoreActions = false;
71   static boolean hasTokenActions = false;
72   static boolean hasSpecial = false;
73   static boolean hasSkip = false;
74   static boolean hasMore = false;
75   static boolean hasToken = false;
76   public static RegularExpression curRE;
77   public static boolean keepLineCol;
78
79   static void PrintClassHead()
80   {
81      int i, j;
82
83      try {
84        File tmp = new File(Options.getOutputDirectory(), tokMgrClassName + ".java");
85        ostr = new java.io.PrintWriter JavaDoc(
86                  new java.io.BufferedWriter JavaDoc(
87                     new java.io.FileWriter JavaDoc(tmp),
88                     8092
89                  )
90               );
91        Vector tn = (Vector)(toolNames.clone());
92        tn.addElement(toolName);
93
94        ostr.println("/* " + getIdString(tn, tokMgrClassName + ".java") + " */");
95
96        int l = 0, kind;
97        i = 1;
98        for (;;)
99        {
100         if (cu_to_insertion_point_1.size() <= l)
101          break;
102
103         kind = ((Token)cu_to_insertion_point_1.elementAt(l)).kind;
104         if(kind == PACKAGE || kind == IMPORT) {
105          for (; i < cu_to_insertion_point_1.size(); i++) {
106            kind = ((Token)cu_to_insertion_point_1.elementAt(i)).kind;
107            if (kind == SEMICOLON ||
108                kind == ABSTRACT ||
109                kind == FINAL ||
110                kind == PUBLIC ||
111                kind == CLASS ||
112                kind == INTERFACE)
113            {
114              cline = ((Token)(cu_to_insertion_point_1.elementAt(l))).beginLine;
115              ccol = ((Token)(cu_to_insertion_point_1.elementAt(l))).beginColumn;
116              for (j = l; j < i; j++) {
117                printToken((Token)(cu_to_insertion_point_1.elementAt(j)), ostr);
118              }
119              if (kind == SEMICOLON)
120                printToken((Token)(cu_to_insertion_point_1.elementAt(j)), ostr);
121              ostr.println("");
122              break;
123            }
124          }
125          l = ++i;
126         }
127         else
128          break;
129        }
130
131        ostr.println("");
132        ostr.println("public class " + tokMgrClassName + " implements " +
133                     cu_name + "Constants");
134        ostr.println("{"); // }
135
}
136      catch (java.io.IOException JavaDoc err) {
137       JavaCCErrors.semantic_error("Could not create file : " + tokMgrClassName + ".java\n");
138       throw new Error JavaDoc();
139     }
140
141     if (token_mgr_decls != null && token_mgr_decls.size() > 0)
142     {
143        Token t = (Token)token_mgr_decls.elementAt(0);
144        boolean commonTokenActionSeen = false;
145        boolean commonTokenActionNeeded = Options.getCommonTokenAction();
146
147        printTokenSetup((Token)token_mgr_decls.elementAt(0));
148        ccol = 1;
149
150        for (j = 0; j < token_mgr_decls.size(); j++)
151        {
152           t = (Token)token_mgr_decls.elementAt(j);
153           if (t.kind == IDENTIFIER &&
154               commonTokenActionNeeded &&
155               !commonTokenActionSeen)
156              commonTokenActionSeen = t.image.equals("CommonTokenAction");
157
158           printToken(t, ostr);
159        }
160
161        ostr.println("");
162        if (commonTokenActionNeeded && !commonTokenActionSeen)
163           JavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. But it appears you have not defined the method :\n"+
164                           " " + staticString + "void CommonTokenAction(Token t)\n" +
165                           "in your TOKEN_MGR_DECLS. The generated token manager will not compile.");
166  
167     }
168     else if (Options.getCommonTokenAction())
169     {
170        JavaCCErrors.warning("You have the COMMON_TOKEN_ACTION option set. But you have not defined the method :\n"+
171                           " " + staticString + "void CommonTokenAction(Token t)\n" +
172                           "in your TOKEN_MGR_DECLS. The generated token manager will not compile.");
173     }
174  
175     ostr.println(" public " + staticString + " java.io.PrintStream debugStream = System.out;");
176     ostr.println(" public " + staticString + " void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }");
177
178     if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
179        ostr.println(" public " + cu_name + " parser = null;");
180     }
181   }
182
183   static void DumpDebugMethods()
184   {
185
186     ostr.println(" " + staticString + " int kindCnt = 0;");
187     ostr.println(" protected " + staticString + " final String jjKindsForBitVector(int i, long vec)");
188     ostr.println(" {");
189     ostr.println(" String retVal = \"\";");
190     ostr.println(" if (i == 0)");
191     ostr.println(" kindCnt = 0;");
192     ostr.println(" for (int j = 0; j < 64; j++)");
193     ostr.println(" {");
194     ostr.println(" if ((vec & (1L << j)) != 0L)");
195     ostr.println(" {");
196     ostr.println(" if (kindCnt++ > 0)");
197     ostr.println(" retVal += \", \";");
198     ostr.println(" if (kindCnt % 5 == 0)");
199     ostr.println(" retVal += \"\\n \";");
200     ostr.println(" retVal += tokenImage[i * 64 + j];");
201     ostr.println(" }");
202     ostr.println(" }");
203     ostr.println(" return retVal;");
204     ostr.println(" }");
205     ostr.println("");
206
207     ostr.println(" protected " + staticString + " final String jjKindsForStateVector(int lexState, int[] vec, int start, int end)");
208     ostr.println(" {");
209     ostr.println(" boolean[] kindDone = new boolean[" + maxOrdinal + "];");
210     ostr.println(" String retVal = \"\";");
211     ostr.println(" int cnt = 0;");
212     ostr.println(" for (int i = start; i < end; i++)");
213     ostr.println(" {");
214     ostr.println(" if (vec[i] == -1)");
215     ostr.println(" continue;");
216     ostr.println(" int[] stateSet = statesForState[curLexState][vec[i]];");
217     ostr.println(" for (int j = 0; j < stateSet.length; j++)");
218     ostr.println(" {");
219     ostr.println(" int state = stateSet[j];");
220     ostr.println(" if (!kindDone[kindForState[lexState][state]])");
221     ostr.println(" {");
222     ostr.println(" kindDone[kindForState[lexState][state]] = true;");
223     ostr.println(" if (cnt++ > 0)");
224     ostr.println(" retVal += \", \";");
225     ostr.println(" if (cnt % 5 == 0)");
226     ostr.println(" retVal += \"\\n \";");
227     ostr.println(" retVal += tokenImage[kindForState[lexState][state]];");
228     ostr.println(" }");
229     ostr.println(" }");
230     ostr.println(" }");
231     ostr.println(" if (cnt == 0)");
232     ostr.println(" return \"{ }\";");
233     ostr.println(" else");
234     ostr.println(" return \"{ \" + retVal + \" }\";");
235     ostr.println(" }");
236     ostr.println("");
237   }
238
239   static void BuildLexStatesTable()
240   {
241      Enumeration e = rexprlist.elements();
242      TokenProduction tp;
243      int i;
244
245      String JavaDoc[] tmpLexStateName = new String JavaDoc[lexstate_I2S.size()];
246      while (e.hasMoreElements())
247      {
248         tp = (TokenProduction)e.nextElement();
249         Vector respecs = tp.respecs;
250         Vector tps;
251
252         for (i = 0; i < tp.lexStates.length; i++)
253         {
254            if ((tps = (Vector)allTpsForState.get(tp.lexStates[i])) == null)
255            {
256               tmpLexStateName[maxLexStates++] = tp.lexStates[i];
257               allTpsForState.put(tp.lexStates[i], tps = new Vector());
258            }
259
260            tps.addElement(tp);
261         }
262
263         if (respecs == null || respecs.size() == 0)
264            continue;
265
266         RegularExpression re;
267         for (i = 0; i < respecs.size(); i++)
268            if (maxOrdinal <= (re = ((RegExprSpec)respecs.elementAt(i)).rexp).ordinal)
269               maxOrdinal = re.ordinal + 1;
270      }
271
272      kinds = new int[maxOrdinal];
273      toSkip = new long[maxOrdinal / 64 + 1];
274      toSpecial = new long[maxOrdinal / 64 + 1];
275      toMore = new long[maxOrdinal / 64 + 1];
276      toToken = new long[maxOrdinal / 64 + 1];
277      toToken[0] = 1L;
278      actions = new Action[maxOrdinal];
279      actions[0] = actForEof;
280      hasTokenActions = actForEof != null;
281      initStates = new Hashtable();
282      canMatchAnyChar = new int[maxLexStates];
283      canLoop = new boolean[maxLexStates];
284      stateHasActions = new boolean[maxLexStates];
285      lexStateName = new String JavaDoc[maxLexStates];
286      singlesToSkip = new NfaState[maxLexStates];
287      System.arraycopy(tmpLexStateName, 0, lexStateName, 0, maxLexStates);
288
289      for (i = 0; i < maxLexStates; i++)
290         canMatchAnyChar[i] = -1;
291
292      hasNfa = new boolean[maxLexStates];
293      mixed = new boolean[maxLexStates];
294      maxLongsReqd = new int[maxLexStates];
295      initMatch = new int[maxLexStates];
296      newLexState = new String JavaDoc[maxOrdinal];
297      newLexState[0] = nextStateForEof;
298      hasEmptyMatch = false;
299      lexStates = new int[maxOrdinal];
300      ignoreCase = new boolean[maxOrdinal];
301      rexprs = new RegularExpression[maxOrdinal];
302      RStringLiteral.allImages = new String JavaDoc[maxOrdinal];
303      canReachOnMore = new boolean[maxLexStates];
304   }
305
306   static int GetIndex(String JavaDoc name)
307   {
308      for (int i = 0; i < lexStateName.length; i++)
309        if (lexStateName[i] != null && lexStateName[i].equals(name))
310           return i;
311
312      throw new Error JavaDoc(); // Should never come here
313
}
314
315   public static void AddCharToSkip(char c, int kind)
316   {
317      singlesToSkip[lexStateIndex].AddChar(c);
318      singlesToSkip[lexStateIndex].kind = kind;
319   }
320
321   public static void start()
322   {
323      if (!Options.getBuildTokenManager() ||
324          Options.getUserTokenManager() ||
325          JavaCCErrors.get_error_count() > 0)
326         return;
327
328      keepLineCol = Options.getKeepLineColumn();
329      Vector choices = new Vector();
330      Enumeration e;
331      TokenProduction tp;
332      int i, j;
333
334      staticString = (Options.getStatic() ? "static " : "");
335      tokMgrClassName = cu_name + "TokenManager";
336
337      PrintClassHead();
338      BuildLexStatesTable();
339
340      e = allTpsForState.keys();
341
342      boolean ignoring = false;
343  
344      while (e.hasMoreElements())
345      {
346         NfaState.ReInit();
347         RStringLiteral.ReInit();
348
349         String JavaDoc key = (String JavaDoc)e.nextElement();
350
351         lexStateIndex = GetIndex(key);
352         lexStateSuffix = "_" + lexStateIndex;
353         Vector allTps = (Vector)allTpsForState.get(key);
354         initStates.put(key, initialState = new NfaState());
355         ignoring = false;
356
357         singlesToSkip[lexStateIndex] = new NfaState();
358         singlesToSkip[lexStateIndex].dummy = true;
359
360         if (key.equals("DEFAULT"))
361            defaultLexState = lexStateIndex;
362
363         for (i = 0; i < allTps.size(); i++)
364         {
365            tp = (TokenProduction)allTps.elementAt(i);
366            int kind = tp.kind;
367            boolean ignore = tp.ignoreCase;
368            Vector rexps = tp.respecs;
369
370            if (i == 0)
371               ignoring = ignore;
372
373            for (j = 0; j < rexps.size(); j++)
374            {
375               RegExprSpec respec = (RegExprSpec)rexps.elementAt(j);
376               curRE = respec.rexp;
377
378               rexprs[curKind = curRE.ordinal] = curRE;
379               lexStates[curRE.ordinal] = lexStateIndex;
380               ignoreCase[curRE.ordinal] = ignore;
381
382               if (curRE.private_rexp)
383               {
384                  kinds[curRE.ordinal] = -1;
385                  continue;
386               }
387
388               if (curRE instanceof RStringLiteral &&
389                   !((RStringLiteral)curRE).image.equals(""))
390               {
391                  ((RStringLiteral)curRE).GenerateDfa(ostr, curRE.ordinal);
392                  if (i != 0 && !mixed[lexStateIndex] && ignoring != ignore)
393                     mixed[lexStateIndex] = true;
394               }
395               else if (curRE.CanMatchAnyChar())
396               {
397                  if (canMatchAnyChar[lexStateIndex] == -1 ||
398                      canMatchAnyChar[lexStateIndex] > curRE.ordinal)
399                     canMatchAnyChar[lexStateIndex] = curRE.ordinal;
400               }
401               else
402               {
403                  Nfa temp;
404
405                  if (curRE instanceof RChoice)
406                     choices.addElement(curRE);
407
408                  temp = curRE.GenerateNfa(ignore);
409                  temp.end.isFinal = true;
410                  temp.end.kind = curRE.ordinal;
411                  initialState.AddMove(temp.start);
412               }
413
414               if (kinds.length < curRE.ordinal)
415               {
416                  int[] tmp = new int[curRE.ordinal + 1];
417
418                  System.arraycopy(kinds, 0, tmp, 0, kinds.length);
419                  kinds = tmp;
420               }
421               //System.out.println(" ordina : " + curRE.ordinal);
422

423               kinds[curRE.ordinal] = kind;
424
425               if (respec.nextState != null &&
426                   !respec.nextState.equals(lexStateName[lexStateIndex]))
427                  newLexState[curRE.ordinal] = respec.nextState;
428
429               if (respec.act != null && respec.act.action_tokens != null &&
430                   respec.act.action_tokens.size() > 0)
431                  actions[curRE.ordinal] = respec.act;
432
433               switch(kind)
434               {
435                  case TokenProduction.SPECIAL :
436                     hasSkipActions |= (actions[curRE.ordinal] != null) ||
437                                       (newLexState[curRE.ordinal] != null);
438                     hasSpecial = true;
439                     toSpecial[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
440                     toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
441                     break;
442                  case TokenProduction.SKIP :
443                     hasSkipActions |= (actions[curRE.ordinal] != null);
444                     hasSkip = true;
445                     toSkip[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
446                     break;
447                  case TokenProduction.MORE :
448                     hasMoreActions |= (actions[curRE.ordinal] != null);
449                     hasMore = true;
450                     toMore[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
451
452                     if (newLexState[curRE.ordinal] != null)
453                        canReachOnMore[GetIndex(newLexState[curRE.ordinal])] = true;
454                     else
455                        canReachOnMore[lexStateIndex] = true;
456
457                     break;
458                  case TokenProduction.TOKEN :
459                     hasTokenActions |= (actions[curRE.ordinal] != null);
460                     hasToken = true;
461                     toToken[curRE.ordinal / 64] |= 1L << (curRE.ordinal % 64);
462                     break;
463               }
464            }
465         }
466
467         // Generate a static block for initializing the nfa transitions
468
NfaState.ComputeClosures();
469
470         for (i = 0; i < initialState.epsilonMoves.size(); i++)
471            ((NfaState)initialState.epsilonMoves.elementAt(i)).GenerateCode();
472
473         if (hasNfa[lexStateIndex] = (NfaState.generatedStates != 0))
474         {
475            initialState.GenerateCode();
476            initialState.GenerateInitMoves(ostr);
477         }
478
479         if (initialState.kind != Integer.MAX_VALUE && initialState.kind != 0)
480         {
481            if ((toSkip[initialState.kind / 64] & (1L << initialState.kind)) != 0L ||
482                (toSpecial[initialState.kind / 64] & (1L << initialState.kind)) != 0L)
483               hasSkipActions = true;
484            else if ((toMore[initialState.kind / 64] & (1L << initialState.kind)) != 0L)
485               hasMoreActions = true;
486            else
487               hasTokenActions = true;
488
489            if (initMatch[lexStateIndex] == 0 ||
490                initMatch[lexStateIndex] > initialState.kind)
491            {
492               initMatch[lexStateIndex] = initialState.kind;
493               hasEmptyMatch = true;
494            }
495         }
496         else if (initMatch[lexStateIndex] == 0)
497            initMatch[lexStateIndex] = Integer.MAX_VALUE;
498
499         RStringLiteral.FillSubString();
500
501         if (hasNfa[lexStateIndex] && !mixed[lexStateIndex])
502            RStringLiteral.GenerateNfaStartStates(ostr, initialState);
503
504         RStringLiteral.DumpDfaCode(ostr);
505
506         if (hasNfa[lexStateIndex])
507            NfaState.DumpMoveNfa(ostr);
508
509         if (stateSetSize < NfaState.generatedStates)
510            stateSetSize = NfaState.generatedStates;
511      }
512
513      for (i = 0; i < choices.size(); i++)
514         ((RChoice)choices.elementAt(i)).CheckUnmatchability();
515
516      NfaState.DumpStateSets(ostr);
517      CheckEmptyStringMatch();
518      NfaState.DumpNonAsciiMoveMethods(ostr);
519      RStringLiteral.DumpStrLiteralImages(ostr);
520      DumpStaticVarDeclarations();
521      DumpFillToken();
522      DumpGetNextToken();
523
524      if (Options.getDebugTokenManager())
525      {
526         NfaState.DumpStatesForKind(ostr);
527         DumpDebugMethods();
528      }
529
530      if (hasLoop)
531      {
532         ostr.println(staticString + "int[] jjemptyLineNo = new int[" + maxLexStates + "];");
533         ostr.println(staticString + "int[] jjemptyColNo = new int[" + maxLexStates + "];");
534         ostr.println(staticString + "boolean[] jjbeenHere = new boolean[" + maxLexStates + "];");
535      }
536
537      if (hasSkipActions)
538         DumpSkipActions();
539      if (hasMoreActions)
540         DumpMoreActions();
541      if (hasTokenActions)
542         DumpTokenActions();
543
544      ostr.println(/*{*/ "}");
545      ostr.close();
546   }
547
548   static void CheckEmptyStringMatch()
549   {
550      int i, j, k, len;
551      boolean[] seen = new boolean[maxLexStates];
552      boolean[] done = new boolean[maxLexStates];
553      String JavaDoc cycle;
554      String JavaDoc reList;
555
556      Outer:
557      for (i = 0; i < maxLexStates; i++)
558      {
559         if (done[i] || initMatch[i] == 0 || initMatch[i] == Integer.MAX_VALUE ||
560             canMatchAnyChar[i] != -1)
561            continue;
562
563         done[i] = true;
564         len = 0;
565         cycle = "";
566         reList = "";
567
568         for (k = 0; k < maxLexStates; k++)
569            seen[k] = false;
570
571         j = i;
572         seen[i] = true;
573         cycle += lexStateName[j] + "-->";
574         while (newLexState[initMatch[j]] != null)
575         {
576            cycle += newLexState[initMatch[j]];
577            if (seen[j = GetIndex(newLexState[initMatch[j]])])
578               break;
579
580            cycle += "-->";
581            done[j] = true;
582            seen[j] = true;
583            if (initMatch[j] == 0 || initMatch[j] == Integer.MAX_VALUE ||
584                canMatchAnyChar[j] != -1)
585               continue Outer;
586            if (len != 0)
587               reList += "; ";
588            reList += "line " + rexprs[initMatch[j]].line + ", column " +
589                      rexprs[initMatch[j]].column;
590            len++;
591         }
592
593         if (newLexState[initMatch[j]] == null)
594            cycle += lexStateName[lexStates[initMatch[j]]];
595
596         for (k = 0; k < maxLexStates; k++)
597            canLoop[k] |= seen[k];
598
599         hasLoop = true;
600         if (len == 0)
601            JavaCCErrors.warning(rexprs[initMatch[i]],
602                 "Regular expression" + ((rexprs[initMatch[i]].label.equals(""))
603                       ? "" : (" for " + rexprs[initMatch[i]].label)) +
604                                 " can be matched by the empty string (\"\") in lexical state " +
605                                 lexStateName[i] + ". This can result in an endless loop of " +
606                                 "empty string matches.");
607                      else
608                      {
609                          JavaCCErrors.warning(rexprs[initMatch[i]],
610                                 "Regular expression" + ((rexprs[initMatch[i]].label.equals(""))
611                                         ? "" : (" for " + rexprs[initMatch[i]].label)) +
612                                 " can be matched by the empty string (\"\") in lexical state " +
613                 lexStateName[i] + ". This regular expression along with the " +
614                 "regular expressions at " + reList + " forms the cycle \n " +
615                 cycle + "\ncontaining regular expressions with empty matches." +
616                 " This can result in an endless loop of empty string matches.");
617         }
618      }
619   }
620
621   static void PrintArrayInitializer(int noElems)
622   {
623      ostr.print("{");
624      for (int i = 0; i < noElems; i++)
625      {
626         if (i % 25 == 0)
627            ostr.print("\n ");
628         ostr.print("0, ");
629      }
630      ostr.println("\n};");
631   }
632
633   static void DumpStaticVarDeclarations()
634   {
635       int i;
636       String JavaDoc charStreamName;
637
638       ostr.println("public static final String[] lexStateNames = {");
639       for (i = 0; i < maxLexStates; i++)
640          ostr.println(" \"" + lexStateName[i] + "\", ");
641       ostr.println("};");
642
643       if (maxLexStates > 1)
644       {
645          ostr.print("public static final int[] jjnewLexState = {");
646
647          for (i = 0; i < maxOrdinal; i++)
648          {
649             if (i % 25 == 0)
650                ostr.print("\n ");
651
652             if (newLexState[i] == null)
653                ostr.print("-1, ");
654             else
655                ostr.print(GetIndex(newLexState[i]) + ", ");
656          }
657          ostr.println("\n};");
658       }
659
660       if (hasSkip || hasMore || hasSpecial)
661       {
662          // Bit vector for TOKEN
663
ostr.print("static final long[] jjtoToken = {");
664          for (i = 0; i < maxOrdinal / 64 + 1; i++)
665          {
666             if (i % 4 == 0)
667                ostr.print("\n ");
668             ostr.print("0x" + Long.toHexString(toToken[i]) + "L, ");
669          }
670          ostr.println("\n};");
671       }
672
673       if (hasSkip || hasSpecial)
674       {
675          // Bit vector for SKIP
676
ostr.print("static final long[] jjtoSkip = {");
677          for (i = 0; i < maxOrdinal / 64 + 1; i++)
678          {
679             if (i % 4 == 0)
680                ostr.print("\n ");
681             ostr.print("0x" + Long.toHexString(toSkip[i]) + "L, ");
682          }
683          ostr.println("\n};");
684       }
685
686       if (hasSpecial)
687       {
688          // Bit vector for SPECIAL
689
ostr.print("static final long[] jjtoSpecial = {");
690          for (i = 0; i < maxOrdinal / 64 + 1; i++)
691          {
692             if (i % 4 == 0)
693                ostr.print("\n ");
694             ostr.print("0x" + Long.toHexString(toSpecial[i]) + "L, ");
695          }
696          ostr.println("\n};");
697       }
698
699       if (hasMore)
700       {
701          // Bit vector for MORE
702
ostr.print("static final long[] jjtoMore = {");
703          for (i = 0; i < maxOrdinal / 64 + 1; i++)
704          {
705             if (i % 4 == 0)
706                ostr.print("\n ");
707             ostr.print("0x" + Long.toHexString(toMore[i]) + "L, ");
708          }
709          ostr.println("\n};");
710       }
711
712       if (Options.getUserCharStream())
713          charStreamName = "CharStream";
714       else
715       {
716          if (Options.getJavaUnicodeEscape())
717             charStreamName = "JavaCharStream";
718          else
719             charStreamName = "SimpleCharStream";
720       }
721
722       ostr.println(staticString + "protected " + charStreamName + " input_stream;");
723
724       ostr.println(staticString + "private final int[] jjrounds = " +
725                       "new int[" + stateSetSize + "];");
726       ostr.println(staticString + "private final int[] jjstateSet = " +
727                       "new int[" + (2 * stateSetSize) + "];");
728
729       if (hasMoreActions || hasSkipActions || hasTokenActions)
730       {
731          ostr.println(staticString + "StringBuffer image;");
732          ostr.println(staticString + "int jjimageLen;");
733          ostr.println(staticString + "int lengthOfMatch;");
734       }
735
736       ostr.println(staticString + "protected char curChar;");
737
738       if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
739          ostr.println("public " + tokMgrClassName + "(" + cu_name + " parserArg, " + charStreamName + " stream){");
740          ostr.println(" parser = parserArg;");
741       } else {
742          ostr.println("public " + tokMgrClassName + "(" + charStreamName + " stream){");
743       }
744
745       if (Options.getStatic() && !Options.getUserCharStream())
746       {
747          ostr.println(" if (input_stream != null)");
748          ostr.println(" throw new TokenMgrError(\"ERROR: Second call to constructor of static lexer. You must use ReInit() to initialize the static variables.\", TokenMgrError.STATIC_LEXER_ERROR);");
749       }
750       else if (!Options.getUserCharStream())
751       {
752          if (Options.getJavaUnicodeEscape())
753             ostr.println(" if (JavaCharStream.staticFlag)");
754          else
755             ostr.println(" if (SimpleCharStream.staticFlag)");
756
757          ostr.println(" throw new Error(\"ERROR: Cannot use a static CharStream class with a non-static lexical analyzer.\");");
758       }
759
760       ostr.println(" input_stream = stream;");
761
762       ostr.println("}");
763
764       if(Options.getTokenManagerUsesParser() && !Options.getStatic()){
765          ostr.println("public " + tokMgrClassName + "(" + cu_name + " parserArg, " + charStreamName + " stream, int lexState){");
766          ostr.println(" this(parserArg, stream);");
767       } else {
768          ostr.println("public " + tokMgrClassName + "(" + charStreamName + " stream, int lexState){");
769          ostr.println(" this(stream);");
770       }
771       ostr.println(" SwitchTo(lexState);");
772       ostr.println("}");
773
774       // Reinit method for reinitializing the parser (for static parsers).
775
ostr.println(staticString + "public void ReInit(" + charStreamName + " stream)");
776       ostr.println("{");
777       ostr.println(" jjmatchedPos = jjnewStateCnt = 0;");
778       ostr.println(" curLexState = defaultLexState;");
779       ostr.println(" input_stream = stream;");
780       ostr.println(" ReInitRounds();");
781       ostr.println("}");
782
783       // Method to reinitialize the jjrounds array.
784
ostr.println(staticString + "private final void ReInitRounds()");
785       ostr.println("{");
786       ostr.println(" int i;");
787       ostr.println(" jjround = 0x" + Integer.toHexString(Integer.MIN_VALUE + 1)+ ";");
788       ostr.println(" for (i = " + stateSetSize + "; i-- > 0;)");
789       ostr.println(" jjrounds[i] = 0x" + Integer.toHexString(Integer.MIN_VALUE) + ";");
790       ostr.println("}");
791
792       // Reinit method for reinitializing the parser (for static parsers).
793
ostr.println(staticString + "public void ReInit(" + charStreamName + " stream, int lexState)");
794       ostr.println("{");
795       ostr.println(" ReInit(stream);");
796       ostr.println(" SwitchTo(lexState);");
797       ostr.println("}");
798
799       ostr.println(staticString + "public void SwitchTo(int lexState)");
800       ostr.println("{");
801       ostr.println(" if (lexState >= " + lexStateName.length + " || lexState < 0)");
802       ostr.println(" throw new TokenMgrError(\"Error: Ignoring invalid lexical state : \"" +
803                      " + lexState + \". State unchanged.\", TokenMgrError.INVALID_LEXICAL_STATE);");
804       ostr.println(" else");
805       ostr.println(" curLexState = lexState;");
806       ostr.println("}");
807
808       ostr.println("");
809   }
810
811   // Assumes l != 0L
812
static char MaxChar(long l)
813   {
814      for (int i = 64; i-- > 0; )
815        if ((l & (1L << i)) != 0L)
816           return (char)i;
817
818      return 0xffff;
819   }
820
821   static void DumpFillToken()
822   {
823      ostr.println(staticString + "protected Token jjFillToken()");
824      ostr.println("{");
825      ostr.println(" Token t = Token.newToken(jjmatchedKind);");
826      ostr.println(" t.kind = jjmatchedKind;");
827
828      if (hasEmptyMatch)
829      {
830         ostr.println(" if (jjmatchedPos < 0)");
831         ostr.println(" {");
832         ostr.println(" if (image == null)");
833         ostr.println(" t.image = \"\";");
834         ostr.println(" else");
835         ostr.println(" t.image = image.toString();");
836
837         if (keepLineCol)
838         {
839            ostr.println(" t.beginLine = t.endLine = input_stream.getBeginLine();");
840            ostr.println(" t.beginColumn = t.endColumn = input_stream.getBeginColumn();");
841         }
842
843         ostr.println(" }");
844         ostr.println(" else");
845         ostr.println(" {");
846         ostr.println(" String im = jjstrLiteralImages[jjmatchedKind];");
847         ostr.println(" t.image = (im == null) ? input_stream.GetImage() : im;");
848
849         if (keepLineCol)
850         {
851            ostr.println(" t.beginLine = input_stream.getBeginLine();");
852            ostr.println(" t.beginColumn = input_stream.getBeginColumn();");
853            ostr.println(" t.endLine = input_stream.getEndLine();");
854            ostr.println(" t.endColumn = input_stream.getEndColumn();");
855         }
856
857         ostr.println(" }");
858      }
859      else
860      {
861         ostr.println(" String im = jjstrLiteralImages[jjmatchedKind];");
862         ostr.println(" t.image = (im == null) ? input_stream.GetImage() : im;");
863
864         if (keepLineCol)
865         {
866            ostr.println(" t.beginLine = input_stream.getBeginLine();");
867            ostr.println(" t.beginColumn = input_stream.getBeginColumn();");
868            ostr.println(" t.endLine = input_stream.getEndLine();");
869            ostr.println(" t.endColumn = input_stream.getEndColumn();");
870         }
871
872      }
873
874      ostr.println(" return t;");
875      ostr.println("}");
876   }
877
878   static void DumpGetNextToken()
879   {
880      int i;
881
882      ostr.println("");
883      ostr.println(staticString + "int curLexState = " + defaultLexState + ";");
884      ostr.println(staticString + "int defaultLexState = " + defaultLexState + ";");
885      ostr.println(staticString + "int jjnewStateCnt;");
886      ostr.println(staticString + "int jjround;");
887      ostr.println(staticString + "int jjmatchedPos;");
888      ostr.println(staticString + "int jjmatchedKind;");
889      ostr.println("");
890      ostr.println("public " + staticString + "Token getNextToken()" +
891                  " ");
892      ostr.println("{");
893      ostr.println(" int kind;");
894      ostr.println(" Token specialToken = null;");
895      ostr.println(" Token matchedToken;");
896      ostr.println(" int curPos = 0;");
897      ostr.println("");
898      ostr.println(" EOFLoop :\n for (;;)");
899      ostr.println(" { ");
900      ostr.println(" try ");
901      ostr.println(" { ");
902      ostr.println(" curChar = input_stream.BeginToken();");
903      ostr.println(" } ");
904      ostr.println(" catch(java.io.IOException e)");
905      ostr.println(" { ");
906
907      if (Options.getDebugTokenManager())
908          ostr.println(" debugStream.println(\"Returning the <EOF> token.\");");
909
910      ostr.println(" jjmatchedKind = 0;");
911      ostr.println(" matchedToken = jjFillToken();");
912
913      if (hasSpecial)
914         ostr.println(" matchedToken.specialToken = specialToken;");
915
916      if (nextStateForEof != null || actForEof != null)
917         ostr.println(" TokenLexicalActions(matchedToken);");
918
919      if (Options.getCommonTokenAction())
920         ostr.println(" CommonTokenAction(matchedToken);");
921
922      ostr.println(" return matchedToken;");
923      ostr.println(" }");
924
925      if (hasMoreActions || hasSkipActions || hasTokenActions)
926      {
927         ostr.println(" image = null;");
928         ostr.println(" jjimageLen = 0;");
929      }
930
931      ostr.println("");
932
933      String JavaDoc prefix = "";
934      if (hasMore)
935      {
936         ostr.println(" for (;;)");
937         ostr.println(" {");
938         prefix = " ";
939      }
940
941      String JavaDoc endSwitch = "";
942      String JavaDoc caseStr = "";
943      // this also sets up the start state of the nfa
944
if (maxLexStates > 1)
945      {
946         ostr.println(prefix + " switch(curLexState)");
947         ostr.println(prefix + " {");
948         endSwitch = prefix + " }";
949         caseStr = prefix + " case ";
950         prefix += " ";
951      }
952
953      prefix += " ";
954      for(i = 0; i < maxLexStates; i++)
955      {
956         if (maxLexStates > 1)
957            ostr.println(caseStr + i + ":");
958
959         if (singlesToSkip[i].HasTransitions())
960         {
961            // added the backup(0) to make JIT happy
962
ostr.println(prefix + "try { input_stream.backup(0);");
963            if (singlesToSkip[i].asciiMoves[0] != 0L &&
964                singlesToSkip[i].asciiMoves[1] != 0L)
965            {
966               ostr.println(prefix + " while ((curChar < 64" + " && (0x" +
967                            Long.toHexString(singlesToSkip[i].asciiMoves[0]) +
968                            "L & (1L << curChar)) != 0L) || \n" +
969                            prefix + " (curChar >> 6) == 1" +
970                            " && (0x" +
971                            Long.toHexString(singlesToSkip[i].asciiMoves[1]) +
972                            "L & (1L << (curChar & 077))) != 0L)");
973            }
974            else if (singlesToSkip[i].asciiMoves[1] == 0L)
975            {
976               ostr.println(prefix + " while (curChar <= " +
977                            (int)MaxChar(singlesToSkip[i].asciiMoves[0]) + " && (0x" +
978                            Long.toHexString(singlesToSkip[i].asciiMoves[0]) +
979                            "L & (1L << curChar)) != 0L)");
980            }
981            else if (singlesToSkip[i].asciiMoves[0] == 0L)
982            {
983               ostr.println(prefix + " while (curChar > 63 && curChar <= " +
984                            ((int)MaxChar(singlesToSkip[i].asciiMoves[1]) + 64) +
985                            " && (0x" +
986                            Long.toHexString(singlesToSkip[i].asciiMoves[1]) +
987                            "L & (1L << (curChar & 077))) != 0L)");
988            }
989
990            if (Options.getDebugTokenManager())
991            {
992               ostr.println(prefix + "{");
993               ostr.println(" debugStream.println(" + (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") + "\"Skipping character : \" + " +
994                  "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \")\");");
995            }
996            ostr.println(prefix + " curChar = input_stream.BeginToken();");
997
998            if (Options.getDebugTokenManager())
999               ostr.println(prefix + "}");
1000
1001           ostr.println(prefix + "}");
1002           ostr.println(prefix + "catch (java.io.IOException e1) { continue EOFLoop; }");
1003        }
1004
1005        if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
1006        {
1007           if (Options.getDebugTokenManager())
1008              ostr.println(" debugStream.println(\" Matched the empty string as \" + tokenImage[" +
1009                initMatch[i] + "] + \" token.\");");
1010
1011           ostr.println(prefix + "jjmatchedKind = " + initMatch[i] + ";");
1012           ostr.println(prefix + "jjmatchedPos = -1;");
1013           ostr.println(prefix + "curPos = 0;");
1014        }
1015        else
1016        {
1017           ostr.println(prefix + "jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
1018           ostr.println(prefix + "jjmatchedPos = 0;");
1019        }
1020
1021     if (Options.getDebugTokenManager())
1022        ostr.println(" debugStream.println(" + (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") + "\"Current character : \" + " +
1023                 "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") at line \" + input_stream.getLine() + \" column \" + input_stream.getColumn());");
1024
1025        ostr.println(prefix + "curPos = jjMoveStringLiteralDfa0_" + i + "();");
1026
1027        if (canMatchAnyChar[i] != -1)
1028        {
1029           if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
1030              ostr.println(prefix + "if (jjmatchedPos < 0 || (jjmatchedPos == 0 && jjmatchedKind > " +
1031                     canMatchAnyChar[i] + "))");
1032           else
1033              ostr.println(prefix + "if (jjmatchedPos == 0 && jjmatchedKind > " +
1034                     canMatchAnyChar[i] + ")");
1035           ostr.println(prefix + "{");
1036           
1037           if (Options.getDebugTokenManager())
1038              ostr.println(" debugStream.println(\" Current character matched as a \" + tokenImage[" +
1039                canMatchAnyChar[i] + "] + \" token.\");");
1040           ostr.println(prefix + " jjmatchedKind = " + canMatchAnyChar[i] + ";");
1041
1042           if (initMatch[i] != Integer.MAX_VALUE && initMatch[i] != 0)
1043              ostr.println(prefix + " jjmatchedPos = 0;");
1044
1045           ostr.println(prefix + "}");
1046        }
1047
1048        if (maxLexStates > 1)
1049           ostr.println(prefix + "break;");
1050     }
1051
1052     if (maxLexStates > 1)
1053        ostr.println(endSwitch);
1054     else if (maxLexStates == 0)
1055        ostr.println(" jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
1056
1057     if (maxLexStates > 1)
1058        prefix = " ";
1059     else
1060        prefix = "";
1061
1062     if (maxLexStates > 0)
1063     {
1064        ostr.println(prefix + " if (jjmatchedKind != 0x" + Integer.toHexString(Integer.MAX_VALUE) + ")");
1065        ostr.println(prefix + " {");
1066        ostr.println(prefix + " if (jjmatchedPos + 1 < curPos)");
1067
1068        if (Options.getDebugTokenManager())
1069        {
1070           ostr.println(prefix + " {");
1071           ostr.println(prefix + " debugStream.println(\" Putting back \" + (curPos - jjmatchedPos - 1) + \" characters into the input stream.\");");
1072        }
1073
1074        ostr.println(prefix + " input_stream.backup(curPos - jjmatchedPos - 1);");
1075
1076        if (Options.getDebugTokenManager())
1077           ostr.println(prefix + " }");
1078
1079        if (Options.getDebugTokenManager())
1080        {
1081           if (Options.getJavaUnicodeEscape() ||
1082               Options.getUserCharStream())
1083              ostr.println(" debugStream.println(\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH (\" + TokenMgrError.addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + \") ******\\n\");");
1084           else
1085              ostr.println(" debugStream.println(\"****** FOUND A \" + tokenImage[jjmatchedKind] + \" MATCH (\" + TokenMgrError.addEscapes(new String(input_stream.GetSuffix(jjmatchedPos + 1))) + \") ******\\n\");");
1086        }
1087
1088        if (hasSkip || hasMore || hasSpecial)
1089        {
1090           ostr.println(prefix + " if ((jjtoToken[jjmatchedKind >> 6] & " +
1091                                     "(1L << (jjmatchedKind & 077))) != 0L)");
1092           ostr.println(prefix + " {");
1093        }
1094
1095        ostr.println(prefix + " matchedToken = jjFillToken();");
1096
1097        if (hasSpecial)
1098           ostr.println(prefix + " matchedToken.specialToken = specialToken;");
1099
1100        if (hasTokenActions)
1101           ostr.println(prefix + " TokenLexicalActions(matchedToken);");
1102
1103        if (maxLexStates > 1)
1104        {
1105           ostr.println(" if (jjnewLexState[jjmatchedKind] != -1)");
1106           ostr.println(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
1107        }
1108
1109        if (Options.getCommonTokenAction())
1110           ostr.println(prefix + " CommonTokenAction(matchedToken);");
1111
1112        ostr.println(prefix + " return matchedToken;");
1113
1114        if (hasSkip || hasMore || hasSpecial)
1115        {
1116           ostr.println(prefix + " }");
1117
1118           if (hasSkip || hasSpecial)
1119           {
1120              if (hasMore)
1121              {
1122                 ostr.println(prefix + " else if ((jjtoSkip[jjmatchedKind >> 6] & " +
1123                                      "(1L << (jjmatchedKind & 077))) != 0L)");
1124              }
1125              else
1126                 ostr.println(prefix + " else");
1127
1128              ostr.println(prefix + " {");
1129
1130              if (hasSpecial)
1131              {
1132                 ostr.println(prefix + " if ((jjtoSpecial[jjmatchedKind >> 6] & " +
1133                                    "(1L << (jjmatchedKind & 077))) != 0L)");
1134                 ostr.println(prefix + " {");
1135
1136                 ostr.println(prefix + " matchedToken = jjFillToken();");
1137
1138                 ostr.println(prefix + " if (specialToken == null)");
1139                 ostr.println(prefix + " specialToken = matchedToken;");
1140                 ostr.println(prefix + " else");
1141                 ostr.println(prefix + " {");
1142                 ostr.println(prefix + " matchedToken.specialToken = specialToken;");
1143                 ostr.println(prefix + " specialToken = (specialToken.next = matchedToken);");
1144                 ostr.println(prefix + " }");
1145
1146                 if (hasSkipActions)
1147                    ostr.println(prefix + " SkipLexicalActions(matchedToken);");
1148
1149                 ostr.println(prefix + " }");
1150
1151                 if (hasSkipActions)
1152                 {
1153                    ostr.println(prefix + " else ");
1154                    ostr.println(prefix + " SkipLexicalActions(null);");
1155                 }
1156              }
1157              else if (hasSkipActions)
1158                 ostr.println(prefix + " SkipLexicalActions(null);");
1159
1160              if (maxLexStates > 1)
1161              {
1162                 ostr.println(" if (jjnewLexState[jjmatchedKind] != -1)");
1163                 ostr.println(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
1164              }
1165
1166              ostr.println(prefix + " continue EOFLoop;");
1167              ostr.println(prefix + " }");
1168           }
1169
1170           if (hasMore)
1171           {
1172              if (hasMoreActions)
1173                 ostr.println(prefix + " MoreLexicalActions();");
1174              else if (hasSkipActions || hasTokenActions)
1175                 ostr.println(prefix + " jjimageLen += jjmatchedPos + 1;");
1176
1177              if (maxLexStates > 1)
1178              {
1179                 ostr.println(" if (jjnewLexState[jjmatchedKind] != -1)");
1180                 ostr.println(prefix + " curLexState = jjnewLexState[jjmatchedKind];");
1181              }
1182              ostr.println(prefix + " curPos = 0;");
1183              ostr.println(prefix + " jjmatchedKind = 0x" + Integer.toHexString(Integer.MAX_VALUE) + ";");
1184
1185              ostr.println(prefix + " try {");
1186              ostr.println(prefix + " curChar = input_stream.readChar();");
1187
1188              if (Options.getDebugTokenManager())
1189                 ostr.println(" debugStream.println(" + (maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") + "\"Current character : \" + " +
1190                 "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") at line \" + input_stream.getLine() + \" column \" + input_stream.getColumn());");
1191              ostr.println(prefix + " continue;");
1192              ostr.println(prefix + " }");
1193              ostr.println(prefix + " catch (java.io.IOException e1) { }");
1194           }
1195        }
1196
1197        ostr.println(prefix + " }");
1198        ostr.println(prefix + " int error_line = input_stream.getEndLine();");
1199        ostr.println(prefix + " int error_column = input_stream.getEndColumn();");
1200        ostr.println(prefix + " String error_after = null;");
1201        ostr.println(prefix + " boolean EOFSeen = false;");
1202        ostr.println(prefix + " try { input_stream.readChar(); input_stream.backup(1); }");
1203        ostr.println(prefix + " catch (java.io.IOException e1) {");
1204        ostr.println(prefix + " EOFSeen = true;");
1205        ostr.println(prefix + " error_after = curPos <= 1 ? \"\" : input_stream.GetImage();");
1206        ostr.println(prefix + " if (curChar == '\\n' || curChar == '\\r') {");
1207        ostr.println(prefix + " error_line++;");
1208        ostr.println(prefix + " error_column = 0;");
1209        ostr.println(prefix + " }");
1210        ostr.println(prefix + " else");
1211        ostr.println(prefix + " error_column++;");
1212        ostr.println(prefix + " }");
1213        ostr.println(prefix + " if (!EOFSeen) {");
1214        ostr.println(prefix + " input_stream.backup(1);");
1215        ostr.println(prefix + " error_after = curPos <= 1 ? \"\" : input_stream.GetImage();");
1216        ostr.println(prefix + " }");
1217        ostr.println(prefix + " throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);");
1218     }
1219
1220     if (hasMore)
1221        ostr.println(prefix + " }");
1222
1223     ostr.println(" }");
1224     ostr.println("}");
1225     ostr.println("");
1226  }
1227
1228  public static void DumpSkipActions()
1229  {
1230     Action act;
1231
1232     ostr.println(staticString + "void SkipLexicalActions(Token matchedToken)");
1233     ostr.println("{");
1234     ostr.println(" switch(jjmatchedKind)");
1235     ostr.println(" {");
1236
1237     Outer:
1238     for (int i = 0; i < maxOrdinal; i++)
1239     {
1240        if ((toSkip[i / 64] & (1L << (i % 64))) == 0L)
1241           continue;
1242
1243        for (;;)
1244        {
1245           if (((act = (Action)actions[i]) == null ||
1246                 act.action_tokens == null ||
1247                 act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
1248              continue Outer;
1249
1250           ostr.println(" case " + i + " :");
1251
1252           if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
1253           {
1254              ostr.println(" if (jjmatchedPos == -1)");
1255              ostr.println(" {");
1256              ostr.println(" if (jjbeenHere[" + lexStates[i] + "] &&");
1257              ostr.println(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
1258              ostr.println(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
1259              ostr.println(" throw new TokenMgrError((\"Error: Bailing out of infinite loop caused by repeated empty string matches at line \" + input_stream.getBeginLine() + \", column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
1260              ostr.println(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
1261              ostr.println(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
1262              ostr.println(" jjbeenHere[" + lexStates[i] + "] = true;");
1263              ostr.println(" }");
1264           }
1265
1266           if ((act = (Action)actions[i]) == null ||
1267                act.action_tokens.size() == 0)
1268              break;
1269
1270           ostr.println(" if (image == null)");
1271           ostr.println(" image = new StringBuffer();");
1272
1273           ostr.print(" image.append");
1274           if (RStringLiteral.allImages[i] != null)
1275              ostr.println("(jjstrLiteralImages[" + i + "]);");
1276           else
1277              if (Options.getJavaUnicodeEscape() ||
1278                  Options.getUserCharStream())
1279                 ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
1280              else
1281                 ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
1282
1283           printTokenSetup((Token)act.action_tokens.elementAt(0));
1284           ccol = 1;
1285
1286           for (int j = 0; j < act.action_tokens.size(); j++)
1287              printToken((Token)act.action_tokens.elementAt(j), ostr);
1288           ostr.println("");
1289
1290           break;
1291        }
1292
1293        ostr.println(" break;");
1294     }
1295
1296     ostr.println(" default :");
1297     ostr.println(" break;");
1298     ostr.println(" }");
1299     ostr.println("}");
1300  }
1301
1302  public static void DumpMoreActions()
1303  {
1304     Action act;
1305
1306     ostr.println(staticString + "void MoreLexicalActions()");
1307     ostr.println("{");
1308     ostr.println(" jjimageLen += (lengthOfMatch = jjmatchedPos + 1);");
1309     ostr.println(" switch(jjmatchedKind)");
1310     ostr.println(" {");
1311
1312     Outer:
1313     for (int i = 0; i < maxOrdinal; i++)
1314     {
1315        if ((toMore[i / 64] & (1L << (i % 64))) == 0L)
1316           continue;
1317
1318        for (;;)
1319        {
1320           if (((act = (Action)actions[i]) == null ||
1321                 act.action_tokens == null ||
1322                 act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
1323              continue Outer;
1324
1325           ostr.println(" case " + i + " :");
1326
1327           if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
1328           {
1329              ostr.println(" if (jjmatchedPos == -1)");
1330              ostr.println(" {");
1331              ostr.println(" if (jjbeenHere[" + lexStates[i] + "] &&");
1332              ostr.println(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
1333              ostr.println(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
1334              ostr.println(" throw new TokenMgrError((\"Error: Bailing out of infinite loop caused by repeated empty string matches at line \" + input_stream.getBeginLine() + \", column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
1335              ostr.println(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
1336              ostr.println(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
1337              ostr.println(" jjbeenHere[" + lexStates[i] + "] = true;");
1338              ostr.println(" }");
1339           }
1340
1341           if ((act = (Action)actions[i]) == null ||
1342                act.action_tokens.size() == 0)
1343           {
1344              break;
1345           }
1346
1347           ostr.println(" if (image == null)");
1348           ostr.println(" image = new StringBuffer();");
1349           ostr.print(" image.append");
1350
1351           if (RStringLiteral.allImages[i] != null)
1352              ostr.println("(jjstrLiteralImages[" + i + "]);");
1353           else
1354              if (Options.getJavaUnicodeEscape() ||
1355                  Options.getUserCharStream())
1356                 ostr.println("(input_stream.GetSuffix(jjimageLen));");
1357              else
1358                 ostr.println("(input_stream.GetSuffix(jjimageLen));");
1359
1360           ostr.println(" jjimageLen = 0;");
1361           printTokenSetup((Token)act.action_tokens.elementAt(0));
1362           ccol = 1;
1363
1364           for (int j = 0; j < act.action_tokens.size(); j++)
1365              printToken((Token)act.action_tokens.elementAt(j), ostr);
1366           ostr.println("");
1367
1368           break;
1369        }
1370
1371        ostr.println(" break;");
1372     }
1373
1374     ostr.println(" default : ");
1375     ostr.println(" break;");
1376
1377     ostr.println(" }");
1378     ostr.println("}");
1379  }
1380
1381  public static void DumpTokenActions()
1382  {
1383     Action act;
1384     int i;
1385
1386     ostr.println(staticString + "void TokenLexicalActions(Token matchedToken)");
1387     ostr.println("{");
1388     ostr.println(" switch(jjmatchedKind)");
1389     ostr.println(" {");
1390
1391     Outer:
1392     for (i = 0; i < maxOrdinal; i++)
1393     {
1394        if ((toToken[i / 64] & (1L << (i % 64))) == 0L)
1395           continue;
1396
1397        for (;;)
1398        {
1399           if (((act = (Action)actions[i]) == null ||
1400                 act.action_tokens == null ||
1401                 act.action_tokens.size() == 0) && !canLoop[lexStates[i]])
1402              continue Outer;
1403
1404           ostr.println(" case " + i + " :");
1405
1406           if (initMatch[lexStates[i]] == i && canLoop[lexStates[i]])
1407           {
1408              ostr.println(" if (jjmatchedPos == -1)");
1409              ostr.println(" {");
1410              ostr.println(" if (jjbeenHere[" + lexStates[i] + "] &&");
1411              ostr.println(" jjemptyLineNo[" + lexStates[i] + "] == input_stream.getBeginLine() && ");
1412              ostr.println(" jjemptyColNo[" + lexStates[i] + "] == input_stream.getBeginColumn())");
1413              ostr.println(" throw new TokenMgrError((\"Error: Bailing out of infinite loop caused by repeated empty string matches at line \" + input_stream.getBeginLine() + \", column \" + input_stream.getBeginColumn() + \".\"), TokenMgrError.LOOP_DETECTED);");
1414              ostr.println(" jjemptyLineNo[" + lexStates[i] + "] = input_stream.getBeginLine();");
1415              ostr.println(" jjemptyColNo[" + lexStates[i] + "] = input_stream.getBeginColumn();");
1416              ostr.println(" jjbeenHere[" + lexStates[i] + "] = true;");
1417              ostr.println(" }");
1418           }
1419
1420           if ((act = (Action)actions[i]) == null ||
1421                act.action_tokens.size() == 0)
1422              break;
1423
1424           if (i == 0)
1425           {
1426              ostr.println(" image = null;"); // For EOF no image is there
1427
}
1428           else
1429           {
1430           ostr.println(" if (image == null)");
1431           ostr.println(" image = new StringBuffer();");
1432           ostr.print(" image.append");
1433
1434           if (RStringLiteral.allImages[i] != null)
1435              ostr.println("(jjstrLiteralImages[" + i + "]);");
1436           else
1437              if (Options.getJavaUnicodeEscape() ||
1438                  Options.getUserCharStream())
1439                 ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
1440              else
1441                 ostr.println("(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));");
1442
1443           }
1444           printTokenSetup((Token)act.action_tokens.elementAt(0));
1445           ccol = 1;
1446
1447           for (int j = 0; j < act.action_tokens.size(); j++)
1448              printToken((Token)act.action_tokens.elementAt(j), ostr);
1449           ostr.println("");
1450
1451           break;
1452        }
1453
1454        ostr.println(" break;");
1455     }
1456
1457     ostr.println(" default : ");
1458     ostr.println(" break;");
1459     ostr.println(" }");
1460     ostr.println("}");
1461  }
1462
1463   public static void reInit()
1464   {
1465      ostr = null;
1466      staticString = null;
1467      tokMgrClassName = null;
1468      allTpsForState = new Hashtable();
1469      lexStateIndex = 0;
1470      kinds = null;
1471      maxOrdinal = 1;
1472      lexStateSuffix = null;
1473      newLexState = null;
1474      lexStates = null;
1475      ignoreCase = null;
1476      actions = null;
1477      initStates = new Hashtable();
1478      stateSetSize = 0;
1479      maxLexStates = 0;
1480      lexStateName = null;
1481      singlesToSkip = null;
1482      toSkip = null;
1483      toSpecial = null;
1484      toMore = null;
1485      toToken = null;
1486      defaultLexState = 0;
1487      rexprs = null;
1488      maxLongsReqd = null;
1489      initMatch = null;
1490      canMatchAnyChar = null;
1491      hasEmptyMatch = false;
1492      canLoop = null;
1493      stateHasActions = null;
1494      hasLoop = false;
1495      canReachOnMore = null;
1496      hasNfa = null;
1497      mixed = null;
1498      initialState = null;
1499      curKind = 0;
1500      hasSkipActions = false;
1501      hasMoreActions = false;
1502      hasTokenActions = false;
1503      hasSpecial = false;
1504      hasSkip = false;
1505      hasMore = false;
1506      hasToken = false;
1507      curRE = null;
1508   }
1509
1510}
1511
Popular Tags