Java > Open Source Codes > org > javacc > parser > RStringLiteral


1 /*
2  * Copyright © 2002 Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
3  * California 95054, U.S.A. All rights reserved. Sun Microsystems, Inc. has
4  * intellectual property rights relating to technology embodied in the product
5  * that is described in this document. In particular, and without limitation,
6  * these intellectual property rights may include one or more of the U.S.
7  * patents listed at http://www.sun.com/patents and one or more additional
8  * patents or pending patent applications in the U.S. and in other countries.
9  * U.S. Government Rights - Commercial software. Government users are subject
10  * to the Sun Microsystems, Inc. standard license agreement and applicable
11  * provisions of the FAR and its supplements. Use is subject to license terms.
12  * Sun, Sun Microsystems, the Sun logo and Java are trademarks or registered
13  * trademarks of Sun Microsystems, Inc. in the U.S. and other countries. This
14  * product is covered and controlled by U.S. Export Control laws and may be
15  * subject to the export or import laws in other countries. Nuclear, missile,
16  * chemical biological weapons or nuclear maritime end uses or end users,
17  * whether direct or indirect, are strictly prohibited. Export or reexport
18  * to countries subject to U.S. embargo or to entities identified on U.S.
19  * export exclusion lists, including, but not limited to, the denied persons
20  * and specially designated nationals lists is strictly prohibited.
21  */
22
23 package org.javacc.parser;
24
25 import java.util.*;
26
27 final class KindInfo
28 {
29    long[] validKinds;
30    long[] finalKinds;
31    int validKindCnt = 0;
32    int finalKindCnt = 0;
33
34    KindInfo(int maxKind)
35    {
36       validKinds = new long[maxKind / 64 + 1];
37       finalKinds = new long[maxKind / 64 + 1];
38    }
39
40    public void InsertValidKind(int kind)
41    {
42       validKinds[kind / 64] |= (1L << (kind % 64));
43       validKindCnt++;
44    }
45
46    public void InsertFinalKind(int kind)
47    {
48       finalKinds[kind / 64] |= (1L << (kind % 64));
49       finalKindCnt++;
50    }
51 };
52
53 /**
54  * Describes string literals.
55  */
56
57 public class RStringLiteral extends RegularExpression {
58
59   /**
60    * The string image of the literal.
61    */
62   public String image;
63
64   static int maxStrKind = 0;
65   static int maxLen = 0;
66   static int charCnt = 0;
67   static Vector charPosKind = new Vector(); // Elements are hashtables
68 // with single char keys;
69 static int[] maxLenForActive = new int[100]; // 6400 tokens
70 public static String [] allImages;
71   static int[][] intermediateKinds;
72   static int[][] intermediateMatchedPos;
73
74   static int startStateCnt = 0;
75   static boolean subString[];
76   static boolean subStringAtPos[];
77   static Hashtable[] statesForPos;
78
79   // Need to call this method after gnerating code for each lexical state. It
80 // initializes all the static variables, so that there is no interference
81 // between the various states of the lexer.
82 public static void ReInit()
83   {
84      maxStrKind = 0;
85      maxLen = 0;
86      charPosKind = new Vector();
87      maxLenForActive = new int[100]; // 6400 tokens
88 intermediateKinds = null;
89      intermediateMatchedPos = null;
90      startStateCnt = 0;
91      subString = null;
92      subStringAtPos = null;
93      statesForPos = null;
94   }
95
96   public static void DumpStrLiteralImages(java.io.PrintWriter ostr)
97   {
98      String image;
99      int charCnt = 0, i;
100
101      ostr.println("public static final String[] jjstrLiteralImages = {");
102
103      if (allImages == null || allImages.length == 0)
104      {
105         ostr.println("};");
106         return;
107      }
108
109      allImages[0] = "";
110      for (i = 0; i < allImages.length; i++)
111      {
112         if ((image = allImages[i]) == null ||
113             ((LexGen.toSkip[i / 64] & (1L << (i % 64))) == 0L &&
114              (LexGen.toMore[i / 64] & (1L << (i % 64))) == 0L &&
115              (LexGen.toToken[i / 64] & (1L << (i % 64))) == 0L) ||
116             (LexGen.toSkip[i / 64] & (1L << (i % 64))) != 0L ||
117             (LexGen.toMore[i / 64] & (1L << (i % 64))) != 0L ||
118             LexGen.canReachOnMore[LexGen.lexStates[i]] ||
119             ((Options.getIgnoreCase() || LexGen.ignoreCase[i]) &&
120              (!image.equals(image.toLowerCase()) ||
121               !image.equals(image.toUpperCase()))))
122         {
123            allImages[i] = null;
124            if ((charCnt += 6) > 80)
125            {
126               ostr.println("");
127               charCnt = 0;
128            }
129
130            ostr.print("null, ");
131            continue;
132         }
133
134         String toPrint = "\"";
135
136         for (int j = 0; j < image.length(); j++)
137         {
138            if (image.charAt(j) <= 0xff)
139               toPrint += ("\\" + Integer.toOctalString((int)image.charAt(j)));
140            else
141            {
142               String hexVal = Integer.toHexString((int)image.charAt(j));
143
144               if (hexVal.length() == 3)
145                  hexVal = "0" + hexVal;
146               toPrint += ("\\u" + hexVal);
147            }
148         }
149
150         toPrint += ( "\", ");
151
152         if ((charCnt += toPrint.length()) >= 80)
153         {
154            ostr.println("");
155            charCnt = 0;
156         }
157
158         ostr.print(toPrint);
159      }
160
161      while (++i < LexGen.maxOrdinal)
162      {
163         if ((charCnt += 6) > 80)
164         {
165            ostr.println("");
166            charCnt = 0;
167         }
168
169         ostr.print("null, ");
170         continue;
171      }
172
173      ostr.println("};");
174   }
175
176   // Used for top level string literals
177 public void GenerateDfa(java.io.PrintWriter ostr, int kind)
178   {
179      String s;
180      Hashtable temp;
181      KindInfo info;
182      int len;
183
184      if (maxStrKind <= ordinal)
185         maxStrKind = ordinal + 1;
186
187      if ((len = image.length()) > maxLen)
188         maxLen = len;
189
190      char c;
191      for (int i = 0; i < len; i++)
192      {
193         if (Options.getIgnoreCase())
194            s = ("" + (c = image.charAt(i))).toLowerCase();
195         else
196            s = "" + (c = image.charAt(i));
197
198         if (!NfaState.unicodeWarningGiven && c > 0xff &&
199             !Options.getJavaUnicodeEscape() &&
200             !Options.getUserCharStream())
201         {
202            NfaState.unicodeWarningGiven = true;
203            JavaCCErrors.warning(LexGen.curRE, "Non-ASCII characters used in regular expression." +
204               "Please make sure you use the correct Reader when you create the parser that can handle your character set.");
205         }
206
207         if (i >= charPosKind.size()) // Kludge, but OK
208 charPosKind.addElement(temp = new Hashtable());
209         else
210            temp = (Hashtable)charPosKind.elementAt(i);
211
212         if ((info = (KindInfo)temp.get(s)) == null)
213            temp.put(s, info = new KindInfo(LexGen.maxOrdinal));
214
215         if (i + 1 == len)
216            info.InsertFinalKind(ordinal);
217         else
218            info.InsertValidKind(ordinal);
219
220         if (!Options.getIgnoreCase() && LexGen.ignoreCase[ordinal] &&
221             c != Character.toLowerCase(c))
222         {
223            s = ("" + image.charAt(i)).toLowerCase();
224
225            if (i >= charPosKind.size()) // Kludge, but OK
226 charPosKind.addElement(temp = new Hashtable());
227            else
228               temp = (Hashtable)charPosKind.elementAt(i);
229
230            if ((info = (KindInfo)temp.get(s)) == null)
231               temp.put(s, info = new KindInfo(LexGen.maxOrdinal));
232
233            if (i + 1 == len)
234               info.InsertFinalKind(ordinal);
235            else
236               info.InsertValidKind(ordinal);
237         }
238
239         if (!Options.getIgnoreCase() && LexGen.ignoreCase[ordinal] &&
240             c != Character.toUpperCase(c))
241         {
242            s = ("" + image.charAt(i)).toUpperCase();
243
244            if (i >= charPosKind.size()) // Kludge, but OK
245 charPosKind.addElement(temp = new Hashtable());
246            else
247               temp = (Hashtable)charPosKind.elementAt(i);
248
249            if ((info = (KindInfo)temp.get(s)) == null)
250               temp.put(s, info = new KindInfo(LexGen.maxOrdinal));
251
252            if (i + 1 == len)
253               info.InsertFinalKind(ordinal);
254            else
255               info.InsertValidKind(ordinal);
256         }
257      }
258
259      maxLenForActive[ordinal / 64] = Math.max(maxLenForActive[ordinal / 64],
260                                                                         len -1);
261      allImages[ordinal] = image;
262   }
263
264   public Nfa GenerateNfa(boolean ignoreCase)
265   {
266      if (image.length() == 1)
267      {
268         RCharacterList temp = new RCharacterList(image.charAt(0));
269         return temp.GenerateNfa(ignoreCase);
270      }
271
272      NfaState startState = new NfaState();
273      NfaState theStartState = startState;
274      NfaState finalState = null;
275
276      if (image.length() == 0)
277         return new Nfa(theStartState, theStartState);
278
279      int i;
280
281      for (i = 0; i < image.length(); i++)
282      {
283         finalState = new NfaState();
284         startState.charMoves = new char[1];
285         startState.AddChar(image.charAt(i));
286
287         if (Options.getIgnoreCase() || ignoreCase)
288         {
289            startState.AddChar(Character.toLowerCase(image.charAt(i)));
290            startState.AddChar(Character.toUpperCase(image.charAt(i)));
291         }
292
293         startState.next = finalState;
294         startState = finalState;
295      }
296
297      return new Nfa(theStartState, finalState);
298   }
299
300   static void DumpNullStrLiterals(java.io.PrintWriter ostr)
301   {
302      ostr.println("{");
303
304      if (NfaState.generatedStates != 0)
305         ostr.println(" return jjMoveNfa" + LexGen.lexStateSuffix + "(" + NfaState.InitStateName() + ", 0);");
306      else
307         ostr.println(" return 1;");
308
309      ostr.println("}");
310   }
311
312   private static int GetStateSetForKind(int pos, int kind)
313   {
314      if (LexGen.mixed[LexGen.lexStateIndex] || NfaState.generatedStates == 0)
315         return -1;
316
317      Hashtable allStateSets = statesForPos[pos];
318
319      if (allStateSets == null)
320         return -1;
321
322      Enumeration e = allStateSets.keys();
323
324      while (e.hasMoreElements())
325      {
326         String s = (String )e.nextElement();
327         long[] actives = (long[])allStateSets.get(s);
328
329         s = s.substring(s.indexOf(", ") + 2);
330         s = s.substring(s.indexOf(", ") + 2);
331
332         if (s.equals("null;"))
333            continue;
334
335         if (actives != null &&
336             (actives[kind / 64] & (1L << (kind % 64))) != 0L)
337         {
338            return NfaState.AddStartStateSet(s);
339         }
340      }
341
342      return -1;
343   }
344
345   static String GetLabel(int kind)
346   {
347      RegularExpression re = LexGen.rexprs[kind];
348
349      if (re instanceof RStringLiteral)
350        return " \"" + JavaCCGlobals.add_escapes(((RStringLiteral)re).image) + "\"";
351      else if (!re.label.equals(""))
352        return " <" + re.label + ">";
353      else
354        return " <token of kind " + kind + ">";
355   }
356
357   static int GetLine(int kind)
358   {
359      return LexGen.rexprs[kind].line;
360   }
361
362   static int GetColumn(int kind)
363   {
364      return LexGen.rexprs[kind].column;
365   }
366
367   /**
368    * Returns true if s1 starts with s2 (ignoring case for each character).
369    */
370   static private boolean StartsWithIgnoreCase(String s1, String s2)
371   {
372      if (s1.length() < s2.length())
373         return false;
374
375      for (int i = 0; i < s2.length(); i++)
376      {
377         char c1 = s1.charAt(i), c2 = s2.charAt(i);
378
379         if (c1 != c2 && Character.toLowerCase(c2) != c1 &&
380             Character.toUpperCase(c2) != c1)
381            return false;
382      }
383
384      return true;
385   }
386
387   static void FillSubString()
388   {
389      String image;
390      subString = new boolean[maxStrKind + 1];
391      subStringAtPos = new boolean[maxLen];
392
393      for (int i = 0; i < maxStrKind; i++)
394      {
395         subString[i] = false;
396
397         if ((image = allImages[i]) == null ||
398             LexGen.lexStates[i] != LexGen.lexStateIndex)
399            continue;
400
401         if (LexGen.mixed[LexGen.lexStateIndex])
402         {
403            // We will not optimize for mixed case
404 subString[i] = true;
405            subStringAtPos[image.length() - 1] = true;
406            continue;
407         }
408
409         for (int j = 0; j < maxStrKind; j++)
410         {
411            if (j != i && LexGen.lexStates[j] == LexGen.lexStateIndex &&
412                ((String )allImages[j]) != null)
413            {
414               if (((String )allImages[j]).indexOf(image) == 0)
415               {
416                  subString[i] = true;
417                  subStringAtPos[image.length() - 1] = true;
418                  break;
419               }
420               else if (Options.getIgnoreCase() &&
421                        StartsWithIgnoreCase((String )allImages[j], image))
422               {
423                  subString[i] = true;
424                  subStringAtPos[image.length() - 1] = true;
425                  break;
426               }
427            }
428         }
429      }
430   }
431
432   static void DumpStartWithStates(java.io.PrintWriter ostr)
433   {
434      ostr.println((Options.getStatic() ? "static " : "") + "private final int " +
435                   "jjStartNfaWithStates" + LexGen.lexStateSuffix + "(int pos, int kind, int state)");
436      ostr.println("{");
437      ostr.println(" jjmatchedKind = kind;");
438      ostr.println(" jjmatchedPos = pos;");
439
440      if (Options.getDebugTokenManager())
441      {
442         ostr.println(" debugStream.println(\" No more string literal token matches are possible.\");");
443         ostr.println(" debugStream.println(\" Currently matched the first \" + (jjmatchedPos + 1) + \" characters as a \" + tokenImage[jjmatchedKind] + \" token.\");");
444      }
445
446      ostr.println(" try { curChar = input_stream.readChar(); }");
447      ostr.println(" catch(java.io.IOException e) { return pos + 1; }");
448
449      if (Options.getDebugTokenManager())
450         ostr.println(" debugStream.println(" + (LexGen.maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") + "\"Current character : \" + " +
451                  "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") at line \" + input_stream.getLine() + \" column \" + input_stream.getColumn());");
452
453      ostr.println(" return jjMoveNfa" + LexGen.lexStateSuffix + "(state, pos + 1);");
454      ostr.println("}");
455   }
456
457   private static boolean boilerPlateDumped = false;
458   static void DumpBoilerPlate(java.io.PrintWriter ostr)
459   {
460      ostr.println((Options.getStatic() ? "static " : "") + "private final int " +
461                   "jjStopAtPos(int pos, int kind)");
462      ostr.println("{");
463      ostr.println(" jjmatchedKind = kind;");
464      ostr.println(" jjmatchedPos = pos;");
465
466      if (Options.getDebugTokenManager())
467      {
468         ostr.println(" debugStream.println(\" No more string literal token matches are possible.\");");
469         ostr.println(" debugStream.println(\" Currently matched the first \" + (jjmatchedPos + 1) + \" characters as a \" + tokenImage[jjmatchedKind] + \" token.\");");
470      }
471
472      ostr.println(" return pos + 1;");
473      ostr.println("}");
474   }
475
476   static String [] ReArrange(Hashtable tab)
477   {
478      String [] ret = new String [tab.size()];
479      Enumeration e = tab.keys();
480      int cnt = 0;
481
482      while (e.hasMoreElements())
483      {
484         int i = 0, j;
485         String s;
486         char c = (s = (String )e.nextElement()).charAt(0);
487
488         while (i < cnt && ret[i].charAt(0) < c) i++;
489
490         if (i < cnt)
491            for (j = cnt - 1; j >= i; j--)
492              ret[j + 1] = ret[j];
493
494         ret[i] = s;
495         cnt++;
496      }
497
498      return ret;
499   }
500
501   static void DumpDfaCode(java.io.PrintWriter ostr)
502   {
503      Hashtable tab;
504      String key;
505      KindInfo info;
506      int maxLongsReqd = maxStrKind / 64 + 1;
507      int i, j, k;
508      boolean ifGenerated;
509      LexGen.maxLongsReqd[LexGen.lexStateIndex] = maxLongsReqd;
510
511      if (maxLen == 0)
512      {
513         ostr.println((Options.getStatic() ? "static " : "") + "private final int " +
514                        "jjMoveStringLiteralDfa0" + LexGen.lexStateSuffix + "()");
515
516         DumpNullStrLiterals(ostr);
517         return;
518      }
519
520      if (!boilerPlateDumped)
521      {
522         DumpBoilerPlate(ostr);
523         boilerPlateDumped = true;
524      }
525
526      if (!LexGen.mixed[LexGen.lexStateIndex] && NfaState.generatedStates != 0)
527         DumpStartWithStates(ostr);
528
529      boolean startNfaNeeded;
530      for (i = 0; i < maxLen; i++)
531      {
532         boolean atLeastOne = false;
533         startNfaNeeded = false;
534         tab = (Hashtable)charPosKind.elementAt(i);
535         String [] keys = ReArrange(tab);
536
537         ostr.print((Options.getStatic() ? "static " : "") + "private final int " +
538                        "jjMoveStringLiteralDfa" + i + LexGen.lexStateSuffix + "(");
539
540         if (i != 0)
541         {
542            if (i == 1)
543            {
544               for (j = 0; j < maxLongsReqd - 1; j++)
545                  if (i <= maxLenForActive[j])
546                  {
547                     if (atLeastOne)
548                        ostr.print(", ");
549                     else
550                        atLeastOne = true;
551                     ostr.print("long active" + j);
552                  }
553
554               if (i <= maxLenForActive[j])
555               {
556                  if (atLeastOne)
557                     ostr.print(", ");
558                  ostr.print("long active" + j);
559               }
560            }
561            else
562            {
563               for (j = 0; j < maxLongsReqd - 1; j++)
564                  if (i <= maxLenForActive[j] + 1)
565                  {
566                     if (atLeastOne)
567                        ostr.print(", ");
568                     else
569                        atLeastOne = true;
570                     ostr.print("long old" + j + ", long active" + j);
571                  }
572
573               if (i <= maxLenForActive[j] + 1)
574               {
575                  if (atLeastOne)
576                     ostr.print(", ");
577                  ostr.print("long old" + j + ", long active" + j);
578               }
579            }
580         }
581         ostr.println(")");
582         ostr.println("{");
583
584         if (i != 0)
585         {
586            if (i > 1)
587            {
588               atLeastOne = false;
589               ostr.print(" if ((");
590
591               for (j = 0; j < maxLongsReqd - 1; j++)
592                  if (i <= maxLenForActive[j] + 1)
593                  {
594                     if (atLeastOne)
595                        ostr.print(" | ");
596                     else
597                        atLeastOne = true;
598                     ostr.print("(active" + j + " &= old" + j + ")");
599                  }
600
601               if (i <= maxLenForActive[j] + 1)
602               {
603                  if (atLeastOne)
604                     ostr.print(" | ");
605                  ostr.print("(active" + j + " &= old" + j + ")");
606               }
607
608               ostr.println(") == 0L)");
609               if (!LexGen.mixed[LexGen.lexStateIndex] && NfaState.generatedStates != 0)
610               {
611                  ostr.print(" return jjStartNfa" + LexGen.lexStateSuffix +
612                                  "(" + (i - 2) + ", ");
613                  for (j = 0; j < maxLongsReqd - 1; j++)
614                     if (i <= maxLenForActive[j] + 1)
615                        ostr.print("old" + j + ", ");
616                     else
617                        ostr.print("0L, ");
618                  if (i <= maxLenForActive[j] + 1)
619                     ostr.println("old" + j + "); ");
620                  else
621                     ostr.println("0L);");
622               }
623               else if (NfaState.generatedStates != 0)
624                  ostr.println(" return jjMoveNfa" + LexGen.lexStateSuffix + "(" + NfaState.InitStateName() + ", " + (i - 1) + ");");
625               else
626                  ostr.println(" return " + i + ";");
627            }
628
629            if (i != 0 && Options.getDebugTokenManager())
630            {
631               ostr.println(" if (jjmatchedKind != 0 && jjmatchedKind != 0x" + Integer.toHexString(Integer.MAX_VALUE) + ")");
632               ostr.println(" debugStream.println(\" Currently matched the first \" + (jjmatchedPos + 1) + \" characters as a \" + tokenImage[jjmatchedKind] + \" token.\");");
633
634               ostr.println(" debugStream.println(\" Possible string literal matches : { \"");
635
636               for (int vecs = 0; vecs < maxStrKind / 64 + 1; vecs++)
637               {
638                  if (i <= maxLenForActive[vecs])
639                  {
640                     ostr.println(" + ");
641                     ostr.print(" jjKindsForBitVector(" + vecs + ", ");
642                     ostr.print("active" + vecs + ") ");
643                  }
644               }
645
646               ostr.println(" + \" } \");");
647            }
648
649            ostr.println(" try { curChar = input_stream.readChar(); }");
650            ostr.println(" catch(java.io.IOException e) {");
651
652            if (!LexGen.mixed[LexGen.lexStateIndex] && NfaState.generatedStates != 0)
653            {
654               ostr.print(" jjStopStringLiteralDfa" + LexGen.lexStateSuffix + "(" + (i - 1) + ", ");
655               for (k = 0; k < maxLongsReqd - 1; k++)
656                  if (i <= maxLenForActive[k])
657                     ostr.print("active" + k + ", ");
658                  else
659                     ostr.print("0L, ");
660
661               if (i <= maxLenForActive[k])
662                  ostr.println("active" + k + ");");
663               else
664                  ostr.println("0L);");
665
666
667               if (i != 0 && Options.getDebugTokenManager())
668               {
669                  ostr.println(" if (jjmatchedKind != 0 && jjmatchedKind != 0x" + Integer.toHexString(Integer.MAX_VALUE) + ")");
670                  ostr.println(" debugStream.println(\" Currently matched the first \" + (jjmatchedPos + 1) + \" characters as a \" + tokenImage[jjmatchedKind] + \" token.\");");
671               }
672               ostr.println(" return " + i + ";");
673            }
674            else if (NfaState.generatedStates != 0)
675               ostr.println(" return jjMoveNfa" + LexGen.lexStateSuffix + "(" + NfaState.InitStateName() + ", " + (i - 1) + ");");
676            else
677               ostr.println(" return " + i + ";");
678
679            ostr.println(" }");
680         }
681
682         if (i != 0 && Options.getDebugTokenManager())
683            ostr.println(" debugStream.println(" + (LexGen.maxLexStates > 1 ? "\"<\" + lexStateNames[curLexState] + \">\" + " : "") + "\"Current character : \" + " +
684                  "TokenMgrError.addEscapes(String.valueOf(curChar)) + \" (\" + (int)curChar + \") at line \" + input_stream.getLine() + \" column \" + input_stream.getColumn());");
685
686         ostr.println(" switch(curChar)");
687         ostr.println(" {");
688
689         CaseLoop:
690         for (int q = 0; q < keys.length; q++)
691         {
692            key = keys[q];
693            info = (KindInfo)tab.get(key);
694            ifGenerated = false;
695            char c = key.charAt(0);
696
697            if (i == 0 && c < 128 && info.finalKindCnt != 0 &&
698                (NfaState.generatedStates == 0 || !NfaState.CanStartNfaUsingAscii(c)))
699            {
700               int kind;
701               Outer:
702               for (j = 0; j < maxLongsReqd; j++)
703                  if (info.finalKinds[j] != 0L)
704                     break;
705
706               for (k = 0; k < 64; k++)
707                  if ((info.finalKinds[j] & (1L << k)) != 0L &&
708                      !subString[kind = (j * 64 + k)])
709                  {
710                    if ((intermediateKinds != null &&
711                         intermediateKinds[(j * 64 + k)] != null &&
712                         intermediateKinds[(j * 64 + k)][i] < (j * 64 + k) &&
713                         intermediateMatchedPos != null &&
714                         intermediateMatchedPos[(j * 64 + k)][i] == i) ||
715                        (LexGen.canMatchAnyChar[LexGen.lexStateIndex] >= 0 &&
716                         LexGen.canMatchAnyChar[LexGen.lexStateIndex] < (j * 64 + k)))
717                       break;
718                     else if ((LexGen.toSkip[kind / 64] & (1L << (kind % 64))) != 0L &&
719                              (LexGen.toSpecial[kind / 64] & (1L << (kind % 64))) == 0L &&
720                              LexGen.actions[kind] == null &&
721                              LexGen.newLexState[kind] == null)
722                     {
723                        LexGen.AddCharToSkip(c, kind);
724
725                        if (Options.getIgnoreCase())
726                        {
727                           if (c != Character.toUpperCase(c))
728                              LexGen.AddCharToSkip(Character.toUpperCase(c), kind);
729
730                           if (c != Character.toLowerCase(c))
731                              LexGen.AddCharToSkip(Character.toLowerCase(c), kind);
732                        }
733                        continue CaseLoop;
734                     }
735                  }
736            }
737
738            // Since we know key is a single character ...
739 if (Options.getIgnoreCase())
740            {
741               if (c != Character.toUpperCase(c))
742                  ostr.println(" case " + (int)Character.toUpperCase(c) + ":");
743
744               if (c != Character.toLowerCase(c))
745                  ostr.println(" case " + (int)Character.toLowerCase(c) + ":");
746            }
747
748            ostr.println(" case " + (int)c + ":");
749
750            long matchedKind;
751            String prefix = (i == 0) ? " " : " ";
752
753            if (info.finalKindCnt != 0)
754            {
755               for (j = 0; j < maxLongsReqd; j++)
756               {
757                  if ((matchedKind = info.finalKinds[j]) == 0L)
758                     continue;
759
760                  for (k = 0; k < 64; k++)
761                  {
762                     if ((matchedKind & (1L << k)) == 0L)
763                        continue;
764
765                     if (ifGenerated)
766                     {
767                        ostr.print(" else if ");
768                     }
769                     else if (i != 0)
770                        ostr.print(" if ");
771
772                     ifGenerated = true;
773
774                     int kindToPrint;
775                     if (i != 0)
776                     {
777                        ostr.println("((active" + j +
778                           " & 0x" + Long.toHexString(1L << k) + "L) != 0L)");
779                     }
780
781                     if (intermediateKinds != null &&
782                         intermediateKinds[(j * 64 + k)] != null &&
783                         intermediateKinds[(j * 64 + k)][i] < (j * 64 + k) &&
784                         intermediateMatchedPos != null &&
785                         intermediateMatchedPos[(j * 64 + k)][i] == i)
786                     {
787                        JavaCCErrors.warning(" \"" +
788                            JavaCCGlobals.add_escapes(allImages[j * 64 + k]) +
789                            "\" cannot be matched as a string literal token " +
790                            "at line " + GetLine(j * 64 + k) + ", column " + GetColumn(j * 64 + k) +
791                            ". It will be matched as " +
792                            GetLabel(intermediateKinds[(j * 64 + k)][i]) + ".");
793                        kindToPrint = intermediateKinds[(j * 64 + k)][i];
794                     }
795                     else if (i == 0 &&
796                          LexGen.canMatchAnyChar[LexGen.lexStateIndex] >= 0 &&
797                          LexGen.canMatchAnyChar[LexGen.lexStateIndex] < (j * 64 + k))
798                     {
799                        JavaCCErrors.warning(" \"" +
800                            JavaCCGlobals.add_escapes(allImages[j * 64 + k]) +
801                            "\" cannot be matched as a string literal token " +
802                            "at line " + GetLine(j * 64 + k) + ", column " + GetColumn(j * 64 + k) +
803                            ". It will be matched as " +
804                            GetLabel(LexGen.canMatchAnyChar[LexGen.lexStateIndex]) + ".");
805                        kindToPrint = LexGen.canMatchAnyChar[LexGen.lexStateIndex];
806                     }
807                     else
808                        kindToPrint = j * 64 + k;
809
810                     if (!subString[(j * 64 + k)])
811                     {
812                        int stateSetName = GetStateSetForKind(i, j * 64 + k);
813
814                        if (stateSetName != -1)
815                        {
816                           ostr.println(prefix + "return jjStartNfaWithStates" +
817                               LexGen.lexStateSuffix + "(" + i +
818                               ", " + kindToPrint + ", " + stateSetName + ");");
819                        }
820                        else
821                           ostr.println(prefix + "return jjStopAtPos" + "(" + i + ", " + kindToPrint + ");");
822                     }
823                     else
824                     {
825                        if ((LexGen.initMatch[LexGen.lexStateIndex] != 0 &&
826                             LexGen.initMatch[LexGen.lexStateIndex] != Integer.MAX_VALUE) ||
827                             i != 0)
828                        {
829                           ostr.println(" {");
830                           ostr.println(prefix + "jjmatchedKind = " +
831                                                      kindToPrint + ";");
832                           ostr.println(prefix + "jjmatchedPos = " + i + ";");
833                           ostr.println(" }");
834                        }
835                        else
836                           ostr.println(prefix + "jjmatchedKind = " +
837                                                      kindToPrint + ";");
838                     }
839                  }
840               }
841            }
842
843            if (info.validKindCnt != 0)
844            {
845               atLeastOne = false;
846
847               if (i == 0)
848               {
849                  ostr.print(" return ");
850
851                  ostr.print("jjMoveStringLiteralDfa" + (i + 1) +
852                                 LexGen.lexStateSuffix + "(");
853                  for (j = 0; j < maxLongsReqd - 1; j++)
854                     if ((i + 1) <= maxLenForActive[j])
855                     {
856                        if (atLeastOne)
857                           ostr.print(", ");
858                        else
859                           atLeastOne = true;
860
861                        ostr.print("0x" + Long.toHexString(info.validKinds[j]) + "L");
862                     }
863
864                  if ((i + 1) <= maxLenForActive[j])
865                  {
866                     if (atLeastOne)
867                        ostr.print(", ");
868
869                     ostr.print("0x" + Long.toHexString(info.validKinds[j]) + "L");
870                  }
871                  ostr.println(");");
872               }
873               else
874               {
875                  ostr.print(" return ");
876
877                  ostr.print("jjMoveStringLiteralDfa" + (i + 1) +
878                                 LexGen.lexStateSuffix + "(");
879
880                  for (j = 0; j < maxLongsReqd - 1; j++)
881                     if ((i + 1) <= maxLenForActive[j] + 1)
882                     {
883                        if (atLeastOne)
884                           ostr.print(", ");
885                        else
886                           atLeastOne = true;
887
888                        if (info.validKinds[j] != 0L)
889                           ostr.print("active" + j + ", 0x" +
890                                   Long.toHexString(info.validKinds[j]) + "L");
891                        else
892                           ostr.print("active" + j + ", 0L");
893                     }
894
895                  if ((i + 1) <= maxLenForActive[j] + 1)
896                  {
897                     if (atLeastOne)
898                        ostr.print(", ");
899                     if (info.validKinds[j] != 0L)
900                        ostr.print("active" + j + ", 0x" +
901                                   Long.toHexString(info.validKinds[j]) + "L");
902                     else
903                        ostr.print("active" + j + ", 0L");
904                  }
905
906                  ostr.println(");");
907               }
908            }
909            else
910            {
911               // A very special case.
912 if (i == 0 && LexGen.mixed[LexGen.lexStateIndex])
913               {
914
915                  if (NfaState.generatedStates != 0)
916                     ostr.println(" return jjMoveNfa" + LexGen.lexStateSuffix + "(" + NfaState.InitStateName() + ", 0);");
917                  else
918                     ostr.println(" return 1;");
919               }
920               else if (i != 0) // No more str literals to look for
921 {
922                  ostr.println(" break;");
923                  startNfaNeeded = true;
924               }
925            }
926         }
927
928         /* default means that the current character is not in any of the
929            strings at this position. */
930         ostr.println(" default :");
931
932         if (Options.getDebugTokenManager())
933            ostr.println(" debugStream.println(\" No string literal matches possible.\");");
934
935         if (NfaState.generatedStates != 0)
936         {
937            if (i == 0)
938            {
939               /* This means no string literal is possible. Just move nfa with
940                  this guy and return. */
941               ostr.println(" return jjMoveNfa" + LexGen.lexStateSuffix + "(" + NfaState.InitStateName() + ", 0);");
942            }
943            else
944            {
945               ostr.println(" break;");
946               startNfaNeeded = true;
947            }
948         }
949         else
950         {
951            ostr.println(" return " + (i + 1) + ";");
952         }
953
954
955         ostr.println(" }");
956
957         if (i != 0)
958         {
959           if (startNfaNeeded)
960           {
961            if (!LexGen.mixed[LexGen.lexStateIndex] && NfaState.generatedStates != 0)
962            {
963               /* Here, a string literal is successfully matched and no more
964                  string literals are possible. So set the kind and state set
965                  upto and including this position for the matched string. */
966
967               ostr.print(" return jjStartNfa" + LexGen.lexStateSuffix + "(" + (i - 1) + ", ");
968               for (k = 0; k < maxLongsReqd - 1; k++)
969                  if (i <= maxLenForActive[k])
970                     ostr.print("active" + k + ", ");
971                  else
972                     ostr.print("0L, ");
973               if (i <= maxLenForActive[k])
974                  ostr.println("active" + k + ");");
975               else
976                  ostr.println("0L);");
977            }
978            else if (NfaState.generatedStates != 0)
979               ostr.println(" return jjMoveNfa" + LexGen.lexStateSuffix + "(" + NfaState.InitStateName() + ", " + i + ");");
980            else
981               ostr.println(" return " + (i + 1) + ";");
982           }
983         }
984
985         ostr.println("}");
986      }
987   }
988
989   static final int GetStrKind(String str)
990   {
991      for (int i = 0; i < maxStrKind; i++)
992      {
993         if (LexGen.lexStates[i] != LexGen.lexStateIndex)
994            continue;
995
996         String image = allImages[i];
997         if (image != null && image.equals(str))
998            return i;
999      }
1000
1001     return Integer.MAX_VALUE;
1002  }
1003
1004  static void GenerateNfaStartStates(java.io.PrintWriter ostr,
1005                                                NfaState initialState)
1006  {
1007     boolean[] seen = new boolean[NfaState.generatedStates];
1008     Hashtable stateSets = new Hashtable();
1009     String stateSetString = "";
1010     int i, j, kind, jjmatchedPos = 0;
1011     int maxKindsReqd = maxStrKind / 64 + 1;
1012     long[] actives;
1013     Vector newStates = new Vector();
1014     Vector oldStates = null, jjtmpStates;
1015
1016     statesForPos = new Hashtable[maxLen];
1017     intermediateKinds = new int[maxStrKind + 1][];
1018     intermediateMatchedPos = new int[maxStrKind + 1][];
1019
1020     for (i = 0; i < maxStrKind; i++)
1021     {
1022        if (LexGen.lexStates[i] != LexGen.lexStateIndex)
1023           continue;
1024
1025        String image = allImages[i];
1026
1027        if (image == null || image.length() < 1)
1028           continue;
1029
1030        try
1031        {
1032           if ((oldStates = (Vector)initialState.epsilonMoves.clone()) == null ||
1033               oldStates.size() == 0)
1034           {
1035              DumpNfaStartStatesCode(statesForPos, ostr);
1036              return;
1037           }
1038        }
1039        catch(Exception e)
1040        {
1041           JavaCCErrors.semantic_error("Error cloning state vector");
1042        }
1043
1044        intermediateKinds[i] = new int[image.length()];
1045        intermediateMatchedPos[i] = new int[image.length()];
1046        jjmatchedPos = 0;
1047        kind = Integer.MAX_VALUE;
1048
1049        for (j = 0; j < image.length(); j++)
1050        {
1051           if (oldStates == null || oldStates.size() <= 0)
1052           {
1053              // Here, j > 0
1054 kind = intermediateKinds[i][j] = intermediateKinds[i][j - 1];
1055              jjmatchedPos = intermediateMatchedPos[i][j] = intermediateMatchedPos[i][j - 1];
1056           }
1057           else
1058           {
1059              kind = NfaState.MoveFromSet(image.charAt(j), oldStates, newStates);
1060              oldStates.removeAllElements();
1061
1062              if (j == 0 && kind != Integer.MAX_VALUE &&
1063                  LexGen.canMatchAnyChar[LexGen.lexStateIndex] != -1 &&
1064                  kind > LexGen.canMatchAnyChar[LexGen.lexStateIndex])
1065                 kind = LexGen.canMatchAnyChar[LexGen.lexStateIndex];
1066
1067              if (GetStrKind(image.substring(0, j + 1)) < kind)
1068              {
1069                 intermediateKinds[i][j] = kind = Integer.MAX_VALUE;
1070                 jjmatchedPos = 0;
1071              }
1072              else if (kind != Integer.MAX_VALUE)
1073              {
1074                 intermediateKinds[i][j] = kind;
1075                 jjmatchedPos = intermediateMatchedPos[i][j] = j;
1076              }
1077              else if (j == 0)
1078                 kind = intermediateKinds[i][j] = Integer.MAX_VALUE;
1079              else
1080              {
1081                 kind = intermediateKinds[i][j] = intermediateKinds[i][j - 1];
1082                 jjmatchedPos = intermediateMatchedPos[i][j] = intermediateMatchedPos[i][j - 1];
1083              }
1084
1085              stateSetString = NfaState.GetStateSetString(newStates);
1086           }
1087
1088           if (kind == Integer.MAX_VALUE &&
1089               (newStates == null || newStates.size() == 0))
1090              continue;
1091
1092           int p;
1093           if (stateSets.get(stateSetString) == null)
1094           {
1095              stateSets.put(stateSetString, stateSetString);
1096              for (p = 0; p < newStates.size(); p++)
1097              {
1098                 if (seen[((NfaState)newStates.elementAt(p)).stateName])
1099                    ((NfaState)newStates.elementAt(p)).inNextOf++;
1100                 else
1101                    seen[((NfaState)newStates.elementAt(p)).stateName] = true;
1102              }
1103           }
1104           else
1105           {
1106              for (p = 0; p < newStates.size(); p++)
1107                 seen[((NfaState)newStates.elementAt(p)).stateName] = true;
1108           }
1109
1110           jjtmpStates = oldStates;
1111           oldStates = newStates;
1112           (newStates = jjtmpStates).removeAllElements();
1113
1114           if (statesForPos[j] == null)
1115              statesForPos[j] = new Hashtable();
1116
1117           if ((actives = ((long[])statesForPos[j].get(kind + ", " +
1118                                    jjmatchedPos + ", " + stateSetString))) == null)
1119           {
1120              actives = new long[maxKindsReqd];
1121              statesForPos[j].put(kind + ", " + jjmatchedPos + ", " +
1122                                                 stateSetString, actives);
1123           }
1124
1125           actives[i / 64] |= 1L << (i % 64);
1126           //String name = NfaState.StoreStateSet(stateSetString);
1127 }
1128     }
1129
1130     DumpNfaStartStatesCode(statesForPos, ostr);
1131  }
1132
1133  static void DumpNfaStartStatesCode(Hashtable[] statesForPos,
1134                                              java.io.PrintWriter ostr)
1135  {
1136      if (maxStrKind == 0) { // No need to generate this function
1137 return;
1138      }
1139
1140     int i, maxKindsReqd = maxStrKind / 64 + 1;
1141     boolean condGenerated = false;
1142     int ind = 0;
1143
1144     ostr.print("private" + (Options.getStatic() ? " static" : "") + " final int jjStopStringLiteralDfa" +
1145                  LexGen.lexStateSuffix + "(int pos, ");
1146     for (i = 0; i < maxKindsReqd - 1; i++)
1147        ostr.print("long active" + i + ", ");
1148     ostr.println("long active" + i + ")\n{");
1149
1150     if (Options.getDebugTokenManager())
1151        ostr.println(" debugStream.println(\" No more string literal token matches are possible.\");");
1152
1153     ostr.println(" switch (pos)\n {");
1154
1155     for (i = 0; i < maxLen - 1; i++)
1156     {
1157        if (statesForPos[i] == null)
1158           continue;
1159
1160        ostr.println(" case " + i + ":");
1161
1162        Enumeration e = statesForPos[i].keys();
1163        while (e.hasMoreElements())
1164        {
1165           String stateSetString = (String )e.nextElement();
1166           long[] actives = (long[])statesForPos[i].get(stateSetString);
1167
1168           for (int j = 0; j < maxKindsReqd; j++)
1169           {
1170              if (actives[j] == 0L)
1171                 continue;
1172
1173              if (condGenerated)
1174                 ostr.print(" || ");
1175              else
1176                 ostr.print(" if (");
1177
1178              condGenerated = true;
1179
1180              ostr.print("(active" + j + " & 0x" +
1181                          Long.toHexString(actives[j]) + "L) != 0L");
1182           }
1183
1184           if (condGenerated)
1185           {
1186              ostr.println(")");
1187
1188              String kindStr = stateSetString.substring(0,
1189                                       ind = stateSetString.indexOf(", "));
1190              String afterKind = stateSetString.substring(ind + 2);
1191              int jjmatchedPos = Integer.parseInt(
1192                           afterKind.substring(0, afterKind.indexOf(", ")));
1193
1194              if (!kindStr.equals(String.valueOf(Integer.MAX_VALUE)))
1195                 ostr.println(" {");
1196
1197              if (!kindStr.equals(String.valueOf(Integer.MAX_VALUE)))
1198              {
1199                 if (i == 0)
1200                 {
1201                    ostr.println(" jjmatchedKind = " + kindStr + ";");
1202
1203                    if ((LexGen.initMatch[LexGen.lexStateIndex] != 0 &&
1204                        LexGen.initMatch[LexGen.lexStateIndex] != Integer.MAX_VALUE))
1205                       ostr.println(" jjmatchedPos = 0;");
1206                 }
1207                 else if (i == jjmatchedPos)
1208                 {
1209                    if (subStringAtPos[i])
1210                    {
1211                       ostr.println(" if (jjmatchedPos != " + i + ")");
1212                       ostr.println(" {");
1213                       ostr.println(" jjmatchedKind = " + kindStr + ";");
1214                       ostr.println(" jjmatchedPos = " + i + ";");
1215                       ostr.println(" }");
1216                    }
1217                    else
1218                    {
1219                       ostr.println(" jjmatchedKind = " + kindStr + ";");
1220                       ostr.println(" jjmatchedPos = " + i + ";");
1221                    }
1222                 }
1223                 else
1224                 {
1225                    if (jjmatchedPos > 0)
1226                       ostr.println(" if (jjmatchedPos < " + jjmatchedPos + ")");
1227                    else
1228                       ostr.println(" if (jjmatchedPos == 0)");
1229                    ostr.println(" {");
1230                    ostr.println(" jjmatchedKind = " + kindStr + ";");
1231                    ostr.println(" jjmatchedPos = " + jjmatchedPos + ";");
1232                    ostr.println(" }");
1233                 }
1234              }
1235
1236              kindStr = stateSetString.substring(0,
1237                                    ind = stateSetString.indexOf(", "));
1238              afterKind = stateSetString.substring(ind + 2);
1239              stateSetString = afterKind.substring(
1240                                       afterKind.indexOf(", ") + 2);
1241
1242              if (stateSetString.equals("null;"))
1243                 ostr.println(" return -1;");
1244              else
1245                 ostr.println(" return " +
1246                    NfaState.AddStartStateSet(stateSetString) + ";");
1247
1248              if (!kindStr.equals(String.valueOf(Integer.MAX_VALUE)))
1249                 ostr.println(" }");
1250              condGenerated = false;
1251           }
1252        }
1253
1254        ostr.println(" return -1;");
1255     }
1256
1257     ostr.println(" default :");
1258     ostr.println(" return -1;");
1259     ostr.println(" }");
1260     ostr.println("}");
1261
1262     ostr.print("private" + (Options.getStatic() ? " static" : "") + " final int jjStartNfa" +
1263                  LexGen.lexStateSuffix + "(int pos, ");
1264     for (i = 0; i < maxKindsReqd - 1; i++)
1265        ostr.print("long active" + i + ", ");
1266     ostr.println("long active" + i + ")\n{");
1267
1268     if (LexGen.mixed[LexGen.lexStateIndex])
1269     {
1270         if (NfaState.generatedStates != 0)
1271            ostr.println(" return jjMoveNfa" + LexGen.lexStateSuffix + "(" + NfaState.InitStateName() + ", pos + 1);");
1272         else
1273            ostr.println(" return pos + 1;");
1274
1275         ostr.println("}");
1276         return;
1277     }
1278
1279     ostr.print(" return jjMoveNfa" + LexGen.lexStateSuffix + "(" +
1280               "jjStopStringLiteralDfa" + LexGen.lexStateSuffix + "(pos, ");
1281     for (i = 0; i < maxKindsReqd - 1; i++)
1282        ostr.print("active" + i + ", ");
1283     ostr.print("active" + i + ")");
1284     ostr.println(", pos + 1);");
1285     ostr.println("}");
1286  }
1287
1288   public static void reInit()
1289   {
1290      maxStrKind = 0;
1291      maxLen = 0;
1292      charCnt = 0;
1293      charPosKind = new Vector();
1294      maxLenForActive = new int[100];
1295      allImages = null;
1296      intermediateKinds = null;
1297      intermediateMatchedPos = null;
1298      startStateCnt = 0;
1299      subString = null;
1300      subStringAtPos = null;
1301      statesForPos = null;
1302      boilerPlateDumped = false;
1303   }
1304
1305}
1306

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags