JavaCodeGenerator


1   package persistence.antlr;
2   
3   /* ANTLR Translator Generator
4    * Project led by Terence Parr at http://www.jGuru.com
5    * Software rights: http://www.antlr.org/license.html
6    *
7    */
8   
9   import java.util.Enumeration  ;
10  import java.util.Hashtable  ;
11  
12  import persistence.antlr.collections.impl.BitSet;
13  import persistence.antlr.collections.impl.Vector;
14  
15  import java.io.PrintWriter  ; //SAS: changed for proper text file io
16  import java.io.IOException  ;
17  import java.io.FileWriter  ;
18  
19  /**Generate MyParser.java, MyLexer.java and MyParserTokenTypes.java */
20  public class JavaCodeGenerator extends CodeGenerator {
21      // non-zero if inside syntactic predicate generation
22      protected int syntacticPredLevel = 0;
23  
24      // Are we generating ASTs (for parsers and tree parsers) right now?
25      protected boolean genAST = false;
26  
27      // Are we saving the text consumed (for lexers) right now?
28      protected boolean saveText = false;
29  
30      // Grammar parameters set up to handle different grammar classes.
31      // These are used to get instanceof tests out of code generation
32      String   labeledElementType;
33      String   labeledElementASTType;
34      String   labeledElementInit;
35      String   commonExtraArgs;
36      String   commonExtraParams;
37      String   commonLocalVars;
38      String   lt1Value;
39      String   exceptionThrown;
40      String   throwNoViable;
41  
42      /** Tracks the rule being generated.  Used for mapTreeId */
43      RuleBlock currentRule;
44  
45      /** Tracks the rule or labeled subrule being generated.  Used for
46       AST generation. */
47      String   currentASTResult;
48  
49      /** Mapping between the ids used in the current alt, and the
50       * names of variables used to represent their AST values.
51       */
52      Hashtable   treeVariableMap = new Hashtable  ();
53  
54      /** Used to keep track of which AST variables have been defined in a rule
55       * (except for the #rule_name and #rule_name_in var's
56       */
57      Hashtable   declaredASTVariables = new Hashtable  ();
58  
59      /* Count of unnamed generated variables */
60      int astVarNumber = 1;
61  
62      /** Special value used to mark duplicate in treeVariableMap */
63      protected static final String   NONUNIQUE = new String  ();
64  
65      public static final int caseSizeThreshold = 127; // ascii is max
66  
67      private Vector semPreds;
68  
69      /** Create a Java code-generator using the given Grammar.
70       * The caller must still call setTool, setBehavior, and setAnalyzer
71       * before generating code.
72       */
73      public JavaCodeGenerator() {
74          super();
75          charFormatter = new JavaCharFormatter();
76      }
77  
78      /** Adds a semantic predicate string to the sem pred vector
79       These strings will be used to build an array of sem pred names
80       when building a debugging parser.  This method should only be
81       called when the debug option is specified
82       */
83      protected int addSemPred(String   predicate) {
84          semPreds.appendElement(predicate);
85          return semPreds.size() - 1;
86      }
87  
88      public void exitIfError() {
89          if (antlrTool.hasError()) {
90              antlrTool.fatalError("Exiting due to errors.");
91          }
92      }
93  
94      /**Generate the parser, lexer, treeparser, and token types in Java */
95      public void gen() {
96          // Do the code generation
97          try {
98              // Loop over all grammars
99              Enumeration   grammarIter = behavior.grammars.elements();
100             while (grammarIter.hasMoreElements()) {
101                 Grammar g = (Grammar)grammarIter.nextElement();
102                 // Connect all the components to each other
103                 g.setGrammarAnalyzer(analyzer);
104                 g.setCodeGenerator(this);
105                 analyzer.setGrammar(g);
106                 // To get right overloading behavior across hetrogeneous grammars
107                 setupGrammarParameters(g);
108                 g.generate();
109                 // print out the grammar with lookahead sets (and FOLLOWs)
110                 // System.out.print(g.toString());
111                 exitIfError();
112             }
113 
114             // Loop over all token managers (some of which are lexers)
115             Enumeration   tmIter = behavior.tokenManagers.elements();
116             while (tmIter.hasMoreElements()) {
117                 TokenManager tm = (TokenManager)tmIter.nextElement();
118                 if (!tm.isReadOnly()) {
119                     // Write the token manager tokens as Java
120                     // this must appear before genTokenInterchange so that
121                     // labels are set on string literals
122                     genTokenTypes(tm);
123                     // Write the token manager tokens as plain text
124                     genTokenInterchange(tm);
125                 }
126                 exitIfError();
127             }
128         }
129         catch (IOException   e) {
130             antlrTool.reportException(e, null);
131         }
132     }
133 
134     /** Generate code for the given grammar element.
135      * @param blk The {...} action to generate
136      */
137     public void gen(ActionElement action) {
138         if (DEBUG_CODE_GENERATOR) System.out.println("genAction(" + action + ")");
139         if (action.isSemPred) {
140             genSemPred(action.actionText, action.line);
141         }
142         else {
143             if (grammar.hasSyntacticPredicate) {
144                 println("if ( inputState.guessing==0 ) {");
145                 tabs++;
146             }
147 
148             // get the name of the followSet for the current rule so that we
149             // can replace $FOLLOW in the .g file.
150             ActionTransInfo tInfo = new ActionTransInfo();
151             String   actionStr = processActionForSpecialSymbols(action.actionText,
152                                                               action.getLine(),
153                                                               currentRule,
154                                                               tInfo);
155 
156             if (tInfo.refRuleRoot != null) {
157                 // Somebody referenced "#rule", make sure translated var is valid
158                 // assignment to #rule is left as a ref also, meaning that assignments
159                 // with no other refs like "#rule = foo();" still forces this code to be
160                 // generated (unnecessarily).
161                 println(tInfo.refRuleRoot + " = (" + labeledElementASTType + ")currentAST.root;");
162             }
163 
164             // dump the translated action
165             printAction(actionStr);
166 
167             if (tInfo.assignToRoot) {
168                 // Somebody did a "#rule=", reset internal currentAST.root
169                 println("currentAST.root = " + tInfo.refRuleRoot + ";");
170                 // reset the child pointer too to be last sibling in sibling list
171                 println("currentAST.child = " + tInfo.refRuleRoot + "!=null &&" + tInfo.refRuleRoot + ".getFirstChild()!=null ?");
172                 tabs++;
173                 println(tInfo.refRuleRoot + ".getFirstChild() : " + tInfo.refRuleRoot + ";");
174                 tabs--;
175                 println("currentAST.advanceChildToEnd();");
176             }
177 
178             if (grammar.hasSyntacticPredicate) {
179                 tabs--;
180                 println("}");
181             }
182         }
183     }
184 
185     /** Generate code for the given grammar element.
186      * @param blk The "x|y|z|..." block to generate
187      */
188     public void gen(AlternativeBlock blk) {
189         if (DEBUG_CODE_GENERATOR) System.out.println("gen(" + blk + ")");
190         println("{");
191         genBlockPreamble(blk);
192         genBlockInitAction(blk);
193 
194         // Tell AST generation to build subrule result
195         String   saveCurrentASTResult = currentASTResult;
196         if (blk.getLabel() != null) {
197             currentASTResult = blk.getLabel();
198         }
199 
200         boolean ok = grammar.theLLkAnalyzer.deterministic(blk);
201 
202         JavaBlockFinishingInfo howToFinish = genCommonBlock(blk, true);
203         genBlockFinish(howToFinish, throwNoViable);
204 
205         println("}");
206 
207         // Restore previous AST generation
208         currentASTResult = saveCurrentASTResult;
209     }
210 
211     /** Generate code for the given grammar element.
212      * @param blk The block-end element to generate.  Block-end
213      * elements are synthesized by the grammar parser to represent
214      * the end of a block.
215      */
216     public void gen(BlockEndElement end) {
217         if (DEBUG_CODE_GENERATOR) System.out.println("genRuleEnd(" + end + ")");
218     }
219 
220     /** Generate code for the given grammar element.
221      * @param blk The character literal reference to generate
222      */
223     public void gen(CharLiteralElement atom) {
224         if (DEBUG_CODE_GENERATOR) System.out.println("genChar(" + atom + ")");
225 
226         if (atom.getLabel() != null) {
227             println(atom.getLabel() + " = " + lt1Value + ";");
228         }
229 
230         boolean oldsaveText = saveText;
231         saveText = saveText && atom.getAutoGenType() == GrammarElement.AUTO_GEN_NONE;
232         genMatch(atom);
233         saveText = oldsaveText;
234     }
235 
236     /** Generate code for the given grammar element.
237      * @param blk The character-range reference to generate
238      */
239     public void gen(CharRangeElement r) {
240         if (r.getLabel() != null && syntacticPredLevel == 0) {
241             println(r.getLabel() + " = " + lt1Value + ";");
242         }
243         boolean flag = ( grammar instanceof LexerGrammar &&
244             ( !saveText ||
245             r.getAutoGenType() ==
246             GrammarElement.AUTO_GEN_BANG ) );
247         if (flag) {
248             println("_saveIndex=text.length();");
249         }
250 
251         println("matchRange(" + r.beginText + "," + r.endText + ");");
252 
253         if (flag) {
254             println("text.setLength(_saveIndex);");
255         }
256     }
257 
258     /** Generate the lexer Java file */
259     public void gen(LexerGrammar g) throws IOException   {
260         // If debugging, create a new sempred vector for this grammar
261         if (g.debuggingOutput)
262             semPreds = new Vector();
263 
264         setGrammar(g);
265         if (!(grammar instanceof LexerGrammar)) {
266             antlrTool.panic("Internal error generating lexer");
267         }
268 
269         // SAS: moved output creation to method so a subclass can change
270         //      how the output is generated (for VAJ interface)
271         setupOutput(grammar.getClassName());
272 
273         genAST = false; // no way to gen trees.
274         saveText = true;    // save consumed characters.
275 
276         tabs = 0;
277 
278         // Generate header common to all Java output files
279         genHeader();
280         // Do not use printAction because we assume tabs==0
281         println(behavior.getHeaderAction(""));
282 
283         // Generate header specific to lexer Java file
284         // println("import java.io.FileInputStream;");
285         println("import java.io.InputStream;");
286         println("import persistence.antlr.TokenStreamException;");
287         println("import persistence.antlr.TokenStreamIOException;");
288         println("import persistence.antlr.TokenStreamRecognitionException;");
289         println("import persistence.antlr.CharStreamException;");
290         println("import persistence.antlr.CharStreamIOException;");
291         println("import persistence.antlr.ANTLRException;");
292         println("import java.io.Reader;");
293         println("import java.util.Hashtable;");
294         println("import persistence.antlr." + grammar.getSuperClass() + ";");
295         println("import persistence.antlr.InputBuffer;");
296         println("import persistence.antlr.ByteBuffer;");
297         println("import persistence.antlr.CharBuffer;");
298         println("import persistence.antlr.Token;");
299         println("import persistence.antlr.CommonToken;");
300         println("import persistence.antlr.RecognitionException;");
301         println("import persistence.antlr.NoViableAltForCharException;");
302         println("import persistence.antlr.MismatchedCharException;");
303         println("import persistence.antlr.TokenStream;");
304         println("import persistence.antlr.ANTLRHashString;");
305         println("import persistence.antlr.LexerSharedInputState;");
306         println("import persistence.antlr.collections.impl.BitSet;");
307         println("import persistence.antlr.SemanticException;");
308 
309         // Generate user-defined lexer file preamble
310         println(grammar.preambleAction.getText());
311 
312         // Generate lexer class definition
313         String   sup = null;
314         if (grammar.superClass != null) {
315             sup = grammar.superClass;
316         }
317         else {
318             sup = "persistence.antlr." + grammar.getSuperClass();
319         }
320 
321         // print javadoc comment if any
322         if (grammar.comment != null) {
323             _println(grammar.comment);
324         }
325 
326         // get prefix (replaces "public" and lets user specify)
327         String   prefix = "public";
328         Token tprefix = (Token)grammar.options.get("classHeaderPrefix");
329         if (tprefix != null) {
330             String   p = StringUtils.stripFrontBack(tprefix.getText(), "\"", "\"");
331             if (p != null) {
332                 prefix = p;
333             }
334         }
335 
336         print(prefix+" ");
337         print("class " + grammar.getClassName() + " extends " + sup);
338         println(" implements " + grammar.tokenManager.getName() + TokenTypesFileSuffix + ", TokenStream");
339         Token tsuffix = (Token)grammar.options.get("classHeaderSuffix");
340         if (tsuffix != null) {
341             String   suffix = StringUtils.stripFrontBack(tsuffix.getText(), "\"", "\"");
342             if (suffix != null) {
343                 print(", " + suffix);   // must be an interface name for Java
344             }
345         }
346         println(" {");
347 
348         // Generate user-defined lexer class members
349         print(
350             processActionForSpecialSymbols(grammar.classMemberAction.getText(), grammar.classMemberAction.getLine(), currentRule, null)
351         );
352 
353         //
354         // Generate the constructor from InputStream, which in turn
355         // calls the ByteBuffer constructor
356         //
357         println("public " + grammar.getClassName() + "(InputStream in) {");
358         tabs++;
359         println("this(new ByteBuffer(in));");
360         tabs--;
361         println("}");
362 
363         //
364         // Generate the constructor from Reader, which in turn
365         // calls the CharBuffer constructor
366         //
367         println("public " + grammar.getClassName() + "(Reader in) {");
368         tabs++;
369         println("this(new CharBuffer(in));");
370         tabs--;
371         println("}");
372 
373         println("public " + grammar.getClassName() + "(InputBuffer ib) {");
374         tabs++;
375         // if debugging, wrap the input buffer in a debugger
376         if (grammar.debuggingOutput)
377             println("this(new LexerSharedInputState(new persistence.antlr.debug.DebuggingInputBuffer(ib)));");
378         else
379             println("this(new LexerSharedInputState(ib));");
380         tabs--;
381         println("}");
382 
383         //
384         // Generate the constructor from InputBuffer (char or byte)
385         //
386         println("public " + grammar.getClassName() + "(LexerSharedInputState state) {");
387         tabs++;
388 
389         println("super(state);");
390         // if debugging, set up array variables and call user-overridable
391         //   debugging setup method
392         if (grammar.debuggingOutput) {
393             println("  ruleNames  = _ruleNames;");
394             println("  semPredNames = _semPredNames;");
395             println("  setupDebugging();");
396         }
397 
398         // Generate the setting of various generated options.
399         // These need to be before the literals since ANTLRHashString depends on
400         // the casesensitive stuff.
401         println("caseSensitiveLiterals = " + g.caseSensitiveLiterals + ";");
402         println("setCaseSensitive(" + g.caseSensitive + ");");
403 
404         // Generate the initialization of a hashtable
405         // containing the string literals used in the lexer
406         // The literals variable itself is in CharScanner
407         println("literals = new Hashtable();");
408         Enumeration   keys = grammar.tokenManager.getTokenSymbolKeys();
409         while (keys.hasMoreElements()) {
410             String   key = (String  )keys.nextElement();
411             if (key.charAt(0) != '"') {
412                 continue;
413             }
414             TokenSymbol sym = grammar.tokenManager.getTokenSymbol(key);
415             if (sym instanceof StringLiteralSymbol) {
416                 StringLiteralSymbol s = (StringLiteralSymbol)sym;
417                 println("literals.put(new ANTLRHashString(" + s.getId() + ", this), new Integer(" + s.getTokenType() + "));");
418             }
419         }
420         tabs--;
421 
422         Enumeration   ids;
423         println("}");
424 
425         // generate the rule name array for debugging
426         if (grammar.debuggingOutput) {
427             println("private static final String _ruleNames[] = {");
428 
429             ids = grammar.rules.elements();
430             int ruleNum = 0;
431             while (ids.hasMoreElements()) {
432                 GrammarSymbol sym = (GrammarSymbol)ids.nextElement();
433                 if (sym instanceof RuleSymbol)
434                     println("  \"" + ((RuleSymbol)sym).getId() + "\",");
435             }
436             println("};");
437         }
438 
439         // Generate nextToken() rule.
440         // nextToken() is a synthetic lexer rule that is the implicit OR of all
441         // user-defined lexer rules.
442         genNextToken();
443 
444         // Generate code for each rule in the lexer
445         ids = grammar.rules.elements();
446         int ruleNum = 0;
447         while (ids.hasMoreElements()) {
448             RuleSymbol sym = (RuleSymbol)ids.nextElement();
449             // Don't generate the synthetic rules
450             if (!sym.getId().equals("mnextToken")) {
451                 genRule(sym, false, ruleNum++);
452             }
453             exitIfError();
454         }
455 
456         // Generate the semantic predicate map for debugging
457         if (grammar.debuggingOutput)
458             genSemPredMap();
459 
460         // Generate the bitsets used throughout the lexer
461         genBitsets(bitsetsUsed, ((LexerGrammar)grammar).charVocabulary.size());
462 
463         println("");
464         println("}");
465 
466         // Close the lexer output stream
467         currentOutput.close();
468         currentOutput = null;
469     }
470 
471     /** Generate code for the given grammar element.
472      * @param blk The (...)+ block to generate
473      */
474     public void gen(OneOrMoreBlock blk) {
475         if (DEBUG_CODE_GENERATOR) System.out.println("gen+(" + blk + ")");
476         String   label;
477         String   cnt;
478         println("{");
479         genBlockPreamble(blk);
480         if (blk.getLabel() != null) {
481             cnt = "_cnt_" + blk.getLabel();
482         }
483         else {
484             cnt = "_cnt" + blk.ID;
485         }
486         println("int " + cnt + "=0;");
487         if (blk.getLabel() != null) {
488             label = blk.getLabel();
489         }
490         else {
491             label = "_loop" + blk.ID;
492         }
493         println(label + ":");
494         println("do {");
495         tabs++;
496         // generate the init action for ()+ ()* inside the loop
497         // this allows us to do usefull EOF checking...
498         genBlockInitAction(blk);
499 
500         // Tell AST generation to build subrule result
501         String   saveCurrentASTResult = currentASTResult;
502         if (blk.getLabel() != null) {
503             currentASTResult = blk.getLabel();
504         }
505 
506         boolean ok = grammar.theLLkAnalyzer.deterministic(blk);
507 
508         // generate exit test if greedy set to false
509         // and an alt is ambiguous with exit branch
510         // or when lookahead derived purely from end-of-file
511         // Lookahead analysis stops when end-of-file is hit,
512         // returning set {epsilon}.  Since {epsilon} is not
513         // ambig with any real tokens, no error is reported
514         // by deterministic() routines and we have to check
515         // for the case where the lookahead depth didn't get
516         // set to NONDETERMINISTIC (this only happens when the
517         // FOLLOW contains real atoms + epsilon).
518         boolean generateNonGreedyExitPath = false;
519         int nonGreedyExitDepth = grammar.maxk;
520 
521         if (!blk.greedy &&
522             blk.exitLookaheadDepth <= grammar.maxk &&
523             blk.exitCache[blk.exitLookaheadDepth].containsEpsilon()) {
524             generateNonGreedyExitPath = true;
525             nonGreedyExitDepth = blk.exitLookaheadDepth;
526         }
527         else if (!blk.greedy &&
528             blk.exitLookaheadDepth == LLkGrammarAnalyzer.NONDETERMINISTIC) {
529             generateNonGreedyExitPath = true;
530         }
531 
532         // generate exit test if greedy set to false
533         // and an alt is ambiguous with exit branch
534         if (generateNonGreedyExitPath) {
535             if (DEBUG_CODE_GENERATOR) {
536                 System.out.println("nongreedy (...)+ loop; exit depth is " +
537                                    blk.exitLookaheadDepth);
538             }
539             String   predictExit =
540                 getLookaheadTestExpression(blk.exitCache,
541                                            nonGreedyExitDepth);
542             println("// nongreedy exit test");
543             println("if ( " + cnt + ">=1 && " + predictExit + ") break " + label + ";");
544         }
545 
546         JavaBlockFinishingInfo howToFinish = genCommonBlock(blk, false);
547         genBlockFinish(
548             howToFinish,
549             "if ( " + cnt + ">=1 ) { break " + label + "; } else {" + throwNoViable + "}"
550         );
551 
552         println(cnt + "++;");
553         tabs--;
554         println("} while (true);");
555         println("}");
556 
557         // Restore previous AST generation
558         currentASTResult = saveCurrentASTResult;
559     }
560 
561     /** Generate the parser Java file */
562     public void gen(ParserGrammar g) throws IOException   {
563 
564         // if debugging, set up a new vector to keep track of sempred
565         //   strings for this grammar
566         if (g.debuggingOutput)
567             semPreds = new Vector();
568 
569         setGrammar(g);
570         if (!(grammar instanceof ParserGrammar)) {
571             antlrTool.panic("Internal error generating parser");
572         }
573 
574         // Open the output stream for the parser and set the currentOutput
575         // SAS: moved file setup so subclass could do it (for VAJ interface)
576         setupOutput(grammar.getClassName());
577 
578         genAST = grammar.buildAST;
579 
580         tabs = 0;
581 
582         // Generate the header common to all output files.
583         genHeader();
584         // Do not use printAction because we assume tabs==0
585         println(behavior.getHeaderAction(""));
586 
587         // Generate header for the parser
588         println("import persistence.antlr.TokenBuffer;");
589         println("import persistence.antlr.TokenStreamException;");
590         println("import persistence.antlr.TokenStreamIOException;");
591         println("import persistence.antlr.ANTLRException;");
592         println("import persistence.antlr." + grammar.getSuperClass() + ";");
593         println("import persistence.antlr.Token;");
594         println("import persistence.antlr.TokenStream;");
595         println("import persistence.antlr.RecognitionException;");
596         println("import persistence.antlr.NoViableAltException;");
597         println("import persistence.antlr.MismatchedTokenException;");
598         println("import persistence.antlr.SemanticException;");
599         println("import persistence.antlr.ParserSharedInputState;");
600         println("import persistence.antlr.collections.impl.BitSet;");
601         if ( genAST ) {
602             println("import persistence.antlr.collections.AST;");
603             println("import java.util.Hashtable;");
604             println("import persistence.antlr.ASTFactory;");
605             println("import persistence.antlr.ASTPair;");
606             println("import persistence.antlr.collections.impl.ASTArray;");
607         }
608 
609         // Output the user-defined parser preamble
610         println(grammar.preambleAction.getText());
611 
612         // Generate parser class definition
613         String   sup = null;
614         if (grammar.superClass != null)
615             sup = grammar.superClass;
616         else
617             sup = "persistence.antlr." + grammar.getSuperClass();
618 
619         // print javadoc comment if any
620         if (grammar.comment != null) {
621             _println(grammar.comment);
622         }
623 
624         // get prefix (replaces "public" and lets user specify)
625         String   prefix = "public";
626         Token tprefix = (Token)grammar.options.get("classHeaderPrefix");
627         if (tprefix != null) {
628             String   p = StringUtils.stripFrontBack(tprefix.getText(), "\"", "\"");
629             if (p != null) {
630                 prefix = p;
631             }
632         }
633 
634         print(prefix+" ");
635         print("class " + grammar.getClassName() + " extends " + sup);
636         println("       implements " + grammar.tokenManager.getName() + TokenTypesFileSuffix);
637 
638         Token tsuffix = (Token)grammar.options.get("classHeaderSuffix");
639         if (tsuffix != null) {
640             String   suffix = StringUtils.stripFrontBack(tsuffix.getText(), "\"", "\"");
641             if (suffix != null)
642                 print(", " + suffix);   // must be an interface name for Java
643         }
644         println(" {");
645 
646         // set up an array of all the rule names so the debugger can
647         // keep track of them only by number -- less to store in tree...
648         if (grammar.debuggingOutput) {
649             println("private static final String _ruleNames[] = {");
650 
651             Enumeration   ids = grammar.rules.elements();
652             int ruleNum = 0;
653             while (ids.hasMoreElements()) {
654                 GrammarSymbol sym = (GrammarSymbol)ids.nextElement();
655                 if (sym instanceof RuleSymbol)
656                     println("  \"" + ((RuleSymbol)sym).getId() + "\",");
657             }
658             println("};");
659         }
660 
661         // Generate user-defined parser class members
662         print(
663             processActionForSpecialSymbols(grammar.classMemberAction.getText(), grammar.classMemberAction.getLine(), currentRule, null)
664         );
665 
666         // Generate parser class constructor from TokenBuffer
667         println("");
668         println("protected " + grammar.getClassName() + "(TokenBuffer tokenBuf, int k) {");
669         println("  super(tokenBuf,k);");
670         println("  tokenNames = _tokenNames;");
671         // if debugging, set up arrays and call the user-overridable
672         //   debugging setup method
673         if (grammar.debuggingOutput) {
674             println("  ruleNames  = _ruleNames;");
675             println("  semPredNames = _semPredNames;");
676             println("  setupDebugging(tokenBuf);");
677         }
678         if ( grammar.buildAST ) {
679             println("  buildTokenTypeASTClassMap();");
680             println("  astFactory = new ASTFactory(getTokenTypeToASTClassMap());");
681         }
682         println("}");
683         println("");
684 
685         println("public " + grammar.getClassName() + "(TokenBuffer tokenBuf) {");
686         println("  this(tokenBuf," + grammar.maxk + ");");
687         println("}");
688         println("");
689 
690         // Generate parser class constructor from TokenStream
691         println("protected " + grammar.getClassName() + "(TokenStream lexer, int k) {");
692         println("  super(lexer,k);");
693         println("  tokenNames = _tokenNames;");
694 
695         // if debugging, set up arrays and call the user-overridable
696         //   debugging setup method
697         if (grammar.debuggingOutput) {
698             println("  ruleNames  = _ruleNames;");
699             println("  semPredNames = _semPredNames;");
700             println("  setupDebugging(lexer);");
701         }
702         if ( grammar.buildAST ) {
703             println("  buildTokenTypeASTClassMap();");
704             println("  astFactory = new ASTFactory(getTokenTypeToASTClassMap());");
705         }
706         println("}");
707         println("");
708 
709         println("public " + grammar.getClassName() + "(TokenStream lexer) {");
710         println("  this(lexer," + grammar.maxk + ");");
711         println("}");
712         println("");
713 
714         println("public " + grammar.getClassName() + "(ParserSharedInputState state) {");
715         println("  super(state," + grammar.maxk + ");");
716         println("  tokenNames = _tokenNames;");
717         if ( grammar.buildAST ) {
718             println("  buildTokenTypeASTClassMap();");
719             println("  astFactory = new ASTFactory(getTokenTypeToASTClassMap());");
720         }
721         println("}");
722         println("");
723 
724         // Generate code for each rule in the grammar
725         Enumeration   ids = grammar.rules.elements();
726         int ruleNum = 0;
727         while (ids.hasMoreElements()) {
728             GrammarSymbol sym = (GrammarSymbol)ids.nextElement();
729             if (sym instanceof RuleSymbol) {
730                 RuleSymbol rs = (RuleSymbol)sym;
731                 genRule(rs, rs.references.size() == 0, ruleNum++);
732             }
733             exitIfError();
734         }
735 
736         // Generate the token names
737         genTokenStrings();
738 
739         if ( grammar.buildAST ) {
740             genTokenASTNodeMap();
741         }
742 
743         // Generate the bitsets used throughout the grammar
744         genBitsets(bitsetsUsed, grammar.tokenManager.maxTokenType());
745 
746         // Generate the semantic predicate map for debugging
747         if (grammar.debuggingOutput)
748             genSemPredMap();
749 
750         // Close class definition
751         println("");
752         println("}");
753 
754         // Close the parser output stream
755         currentOutput.close();
756         currentOutput = null;
757     }
758 
759     /** Generate code for the given grammar element.
760      * @param blk The rule-reference to generate
761      */
762     public void gen(RuleRefElement rr) {
763         if (DEBUG_CODE_GENERATOR) System.out.println("genRR(" + rr + ")");
764         RuleSymbol rs = (RuleSymbol)grammar.getSymbol(rr.targetRule);
765         if (rs == null || !rs.isDefined()) {
766             // Is this redundant???
767             antlrTool.error("Rule '" + rr.targetRule + "' is not defined", grammar.getFilename(), rr.getLine(), rr.getColumn());
768             return;
769         }
770         if (!(rs instanceof RuleSymbol)) {
771             // Is this redundant???
772             antlrTool.error("'" + rr.targetRule + "' does not name a grammar rule", grammar.getFilename(), rr.getLine(), rr.getColumn());
773             return;
774         }
775 
776         genErrorTryForElement(rr);
777 
778         // AST value for labeled rule refs in tree walker.
779         // This is not AST construction;  it is just the input tree node value.
780         if (grammar instanceof TreeWalkerGrammar &&
781             rr.getLabel() != null &&
782             syntacticPredLevel == 0) {
783             println(rr.getLabel() + " = _t==ASTNULL ? null : " + lt1Value + ";");
784         }
785 
786         // if in lexer and ! on rule ref or alt or rule, save buffer index to kill later
787         if (grammar instanceof LexerGrammar && (!saveText || rr.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
788             println("_saveIndex=text.length();");
789         }
790 
791         // Process return value assignment if any
792         printTabs();
793         if (rr.idAssign != null) {
794             // Warn if the rule has no return type
795             if (rs.block.returnAction == null) {
796                 antlrTool.warning("Rule '" + rr.targetRule + "' has no return type", grammar.getFilename(), rr.getLine(), rr.getColumn());
797             }
798             _print(rr.idAssign + "=");
799         }
800         else {
801             // Warn about return value if any, but not inside syntactic predicate
802             if (!(grammar instanceof LexerGrammar) && syntacticPredLevel == 0 && rs.block.returnAction != null) {
803                 antlrTool.warning("Rule '" + rr.targetRule + "' returns a value", grammar.getFilename(), rr.getLine(), rr.getColumn());
804             }
805         }
806 
807         // Call the rule
808         GenRuleInvocation(rr);
809 
810         // if in lexer and ! on element or alt or rule, save buffer index to kill later
811         if (grammar instanceof LexerGrammar && (!saveText || rr.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
812             println("text.setLength(_saveIndex);");
813         }
814 
815         // if not in a syntactic predicate
816         if (syntacticPredLevel == 0) {
817             boolean doNoGuessTest = (
818                 grammar.hasSyntacticPredicate &&
819                 (
820                 grammar.buildAST && rr.getLabel() != null ||
821                 (genAST && rr.getAutoGenType() == GrammarElement.AUTO_GEN_NONE)
822                 )
823                 );
824             if (doNoGuessTest) {
825                 // println("if (inputState.guessing==0) {");
826                 // tabs++;
827             }
828 
829             if (grammar.buildAST && rr.getLabel() != null) {
830                 // always gen variable for rule return on labeled rules
831                 println(rr.getLabel() + "_AST = (" + labeledElementASTType + ")returnAST;");
832             }
833             if (genAST) {
834                 switch (rr.getAutoGenType()) {
835                     case GrammarElement.AUTO_GEN_NONE:
836                         // println("theASTFactory.addASTChild(currentAST, returnAST);");
837                         println("astFactory.addASTChild(currentAST, returnAST);");
838                         break;
839                     case GrammarElement.AUTO_GEN_CARET:
840                         antlrTool.error("Internal: encountered ^ after rule reference");
841                         break;
842                     default:
843                         break;
844                 }
845             }
846 
847             // if a lexer and labeled, Token label defined at rule level, just set it here
848             if (grammar instanceof LexerGrammar && rr.getLabel() != null) {
849                 println(rr.getLabel() + "=_returnToken;");
850             }
851 
852             if (doNoGuessTest) {
853                 // tabs--;
854                 // println("}");
855             }
856         }
857         genErrorCatchForElement(rr);
858     }
859 
860     /** Generate code for the given grammar element.
861      * @param blk The string-literal reference to generate
862      */
863     public void gen(StringLiteralElement atom) {
864         if (DEBUG_CODE_GENERATOR) System.out.println("genString(" + atom + ")");
865 
866         // Variable declarations for labeled elements
867         if (atom.getLabel() != null && syntacticPredLevel == 0) {
868             println(atom.getLabel() + " = " + lt1Value + ";");
869         }
870 
871         // AST
872         genElementAST(atom);
873 
874         // is there a bang on the literal?
875         boolean oldsaveText = saveText;
876         saveText = saveText && atom.getAutoGenType() == GrammarElement.AUTO_GEN_NONE;
877 
878         // matching
879         genMatch(atom);
880 
881         saveText = oldsaveText;
882 
883         // tack on tree cursor motion if doing a tree walker
884         if (grammar instanceof TreeWalkerGrammar) {
885             println("_t = _t.getNextSibling();");
886         }
887     }
888 
889     /** Generate code for the given grammar element.
890      * @param blk The token-range reference to generate
891      */
892     public void gen(TokenRangeElement r) {
893         genErrorTryForElement(r);
894         if (r.getLabel() != null && syntacticPredLevel == 0) {
895             println(r.getLabel() + " = " + lt1Value + ";");
896         }
897 
898         // AST
899         genElementAST(r);
900 
901         // match
902         println("matchRange(" + r.beginText + "," + r.endText + ");");
903         genErrorCatchForElement(r);
904     }
905 
906     /** Generate code for the given grammar element.
907      * @param blk The token-reference to generate
908      */
909     public void gen(TokenRefElement atom) {
910         if (DEBUG_CODE_GENERATOR) System.out.println("genTokenRef(" + atom + ")");
911         if (grammar instanceof LexerGrammar) {
912             antlrTool.panic("Token reference found in lexer");
913         }
914         genErrorTryForElement(atom);
915         // Assign Token value to token label variable
916         if (atom.getLabel() != null && syntacticPredLevel == 0) {
917             println(atom.getLabel() + " = " + lt1Value + ";");
918         }
919 
920         // AST
921         genElementAST(atom);
922         // matching
923         genMatch(atom);
924         genErrorCatchForElement(atom);
925 
926         // tack on tree cursor motion if doing a tree walker
927         if (grammar instanceof TreeWalkerGrammar) {
928             println("_t = _t.getNextSibling();");
929         }
930     }
931 
932     public void gen(TreeElement t) {
933         // save AST cursor
934         println("AST __t" + t.ID + " = _t;");
935 
936         // If there is a label on the root, then assign that to the variable
937         if (t.root.getLabel() != null) {
938             println(t.root.getLabel() + " = _t==ASTNULL ? null :(" + labeledElementASTType + ")_t;");
939         }
940 
941         // check for invalid modifiers ! and ^ on tree element roots
942         if ( t.root.getAutoGenType() == GrammarElement.AUTO_GEN_BANG ) {
943             antlrTool.error("Suffixing a root node with '!' is not implemented",
944                          grammar.getFilename(), t.getLine(), t.getColumn());
945             t.root.setAutoGenType(GrammarElement.AUTO_GEN_NONE);
946         }
947         if ( t.root.getAutoGenType() == GrammarElement.AUTO_GEN_CARET ) {
948             antlrTool.warning("Suffixing a root node with '^' is redundant; already a root",
949                          grammar.getFilename(), t.getLine(), t.getColumn());
950             t.root.setAutoGenType(GrammarElement.AUTO_GEN_NONE);
951         }
952 
953         // Generate AST variables
954         genElementAST(t.root);
955         if (grammar.buildAST) {
956             // Save the AST construction state
957             println("ASTPair __currentAST" + t.ID + " = currentAST.copy();");
958             // Make the next item added a child of the TreeElement root
959             println("currentAST.root = currentAST.child;");
960             println("currentAST.child = null;");
961         }
962 
963         // match root
964         if ( t.root instanceof WildcardElement ) {
965             println("if ( _t==null ) throw new MismatchedTokenException();");
966         }
967         else {
968             genMatch(t.root);
969         }
970         // move to list of children
971         println("_t = _t.getFirstChild();");
972 
973         // walk list of children, generating code for each
974         for (int i = 0; i < t.getAlternatives().size(); i++) {
975             Alternative a = t.getAlternativeAt(i);
976             AlternativeElement e = a.head;
977             while (e != null) {
978                 e.generate();
979                 e = e.next;
980             }
981         }
982 
983         if (grammar.buildAST) {
984             // restore the AST construction state to that just after the
985             // tree root was added
986             println("currentAST = __currentAST" + t.ID + ";");
987         }
988         // restore AST cursor
989         println("_t = __t" + t.ID + ";");
990         // move cursor to sibling of tree just parsed
991         println("_t = _t.getNextSibling();");
992     }
993 
994     /** Generate the tree-parser Java file */
995     public void gen(TreeWalkerGrammar g) throws IOException   {
996         // SAS: debugging stuff removed for now...
997         setGrammar(g);
998         if (!(grammar instanceof TreeWalkerGrammar)) {
999             antlrTool.panic("Internal error generating tree-walker");
1000        }
1001        // Open the output stream for the parser and set the currentOutput
1002        // SAS: move file open to method so subclass can override it
1003        //      (mainly for VAJ interface)
1004        setupOutput(grammar.getClassName());
1005
1006        genAST = grammar.buildAST;
1007        tabs = 0;
1008
1009        // Generate the header common to all output files.
1010        genHeader();
1011        // Do not use printAction because we assume tabs==0
1012        println(behavior.getHeaderAction(""));
1013
1014        // Generate header for the parser
1015        println("import persistence.antlr." + grammar.getSuperClass() + ";");
1016        println("import persistence.antlr.Token;");
1017        println("import persistence.antlr.collections.AST;");
1018        println("import persistence.antlr.RecognitionException;");
1019        println("import persistence.antlr.ANTLRException;");
1020        println("import persistence.antlr.NoViableAltException;");
1021        println("import persistence.antlr.MismatchedTokenException;");
1022        println("import persistence.antlr.SemanticException;");
1023        println("import persistence.antlr.collections.impl.BitSet;");
1024        println("import persistence.antlr.ASTPair;");
1025        println("import persistence.antlr.collections.impl.ASTArray;");
1026
1027        // Output the user-defined parser premamble
1028        println(grammar.preambleAction.getText());
1029
1030        // Generate parser class definition
1031        String   sup = null;
1032        if (grammar.superClass != null) {
1033            sup = grammar.superClass;
1034        }
1035        else {
1036            sup = "persistence.antlr." + grammar.getSuperClass();
1037        }
1038        println("");
1039
1040        // print javadoc comment if any
1041        if (grammar.comment != null) {
1042            _println(grammar.comment);
1043        }
1044
1045        // get prefix (replaces "public" and lets user specify)
1046        String   prefix = "public";
1047        Token tprefix = (Token)grammar.options.get("classHeaderPrefix");
1048        if (tprefix != null) {
1049            String   p = StringUtils.stripFrontBack(tprefix.getText(), "\"", "\"");
1050            if (p != null) {
1051                prefix = p;
1052            }
1053        }
1054
1055        print(prefix+" ");
1056        print("class " + grammar.getClassName() + " extends " + sup);
1057        println("       implements " + grammar.tokenManager.getName() + TokenTypesFileSuffix);
1058        Token tsuffix = (Token)grammar.options.get("classHeaderSuffix");
1059        if (tsuffix != null) {
1060            String   suffix = StringUtils.stripFrontBack(tsuffix.getText(), "\"", "\"");
1061            if (suffix != null) {
1062                print(", " + suffix);   // must be an interface name for Java
1063            }
1064        }
1065        println(" {");
1066
1067        // Generate user-defined parser class members
1068        print(
1069            processActionForSpecialSymbols(grammar.classMemberAction.getText(), grammar.classMemberAction.getLine(), currentRule, null)
1070        );
1071
1072        // Generate default parser class constructor
1073        println("public " + grammar.getClassName() + "() {");
1074        tabs++;
1075        println("tokenNames = _tokenNames;");
1076        tabs--;
1077        println("}");
1078        println("");
1079
1080        // Generate code for each rule in the grammar
1081        Enumeration   ids = grammar.rules.elements();
1082        int ruleNum = 0;
1083        String   ruleNameInits = "";
1084        while (ids.hasMoreElements()) {
1085            GrammarSymbol sym = (GrammarSymbol)ids.nextElement();
1086            if (sym instanceof RuleSymbol) {
1087                RuleSymbol rs = (RuleSymbol)sym;
1088                genRule(rs, rs.references.size() == 0, ruleNum++);
1089            }
1090            exitIfError();
1091        }
1092
1093        // Generate the token names
1094        genTokenStrings();
1095
1096        // Generate the bitsets used throughout the grammar
1097        genBitsets(bitsetsUsed, grammar.tokenManager.maxTokenType());
1098
1099        // Close class definition
1100        println("}");
1101        println("");
1102
1103        // Close the parser output stream
1104        currentOutput.close();
1105        currentOutput = null;
1106    }
1107
1108    /** Generate code for the given grammar element.
1109     * @param wc The wildcard element to generate
1110     */
1111    public void gen(WildcardElement wc) {
1112        // Variable assignment for labeled elements
1113        if (wc.getLabel() != null && syntacticPredLevel == 0) {
1114            println(wc.getLabel() + " = " + lt1Value + ";");
1115        }
1116
1117        // AST
1118        genElementAST(wc);
1119        // Match anything but EOF
1120        if (grammar instanceof TreeWalkerGrammar) {
1121            println("if ( _t==null ) throw new MismatchedTokenException();");
1122        }
1123        else if (grammar instanceof LexerGrammar) {
1124            if (grammar instanceof LexerGrammar &&
1125                (!saveText || wc.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
1126                println("_saveIndex=text.length();");
1127            }
1128            println("matchNot(EOF_CHAR);");
1129            if (grammar instanceof LexerGrammar &&
1130                (!saveText || wc.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
1131                println("text.setLength(_saveIndex);"); // kill text atom put in buffer
1132            }
1133        }
1134        else {
1135            println("matchNot(" + getValueString(Token.EOF_TYPE) + ");");
1136        }
1137
1138        // tack on tree cursor motion if doing a tree walker
1139        if (grammar instanceof TreeWalkerGrammar) {
1140            println("_t = _t.getNextSibling();");
1141        }
1142    }
1143
1144    /** Generate code for the given grammar element.
1145     * @param blk The (...)* block to generate
1146     */
1147    public void gen(ZeroOrMoreBlock blk) {
1148        if (DEBUG_CODE_GENERATOR) System.out.println("gen*(" + blk + ")");
1149        println("{");
1150        genBlockPreamble(blk);
1151        String   label;
1152        if (blk.getLabel() != null) {
1153            label = blk.getLabel();
1154        }
1155        else {
1156            label = "_loop" + blk.ID;
1157        }
1158        println(label + ":");
1159        println("do {");
1160        tabs++;
1161        // generate the init action for ()* inside the loop
1162        // this allows us to do usefull EOF checking...
1163        genBlockInitAction(blk);
1164
1165        // Tell AST generation to build subrule result
1166        String   saveCurrentASTResult = currentASTResult;
1167        if (blk.getLabel() != null) {
1168            currentASTResult = blk.getLabel();
1169        }
1170
1171        boolean ok = grammar.theLLkAnalyzer.deterministic(blk);
1172
1173        // generate exit test if greedy set to false
1174        // and an alt is ambiguous with exit branch
1175        // or when lookahead derived purely from end-of-file
1176        // Lookahead analysis stops when end-of-file is hit,
1177        // returning set {epsilon}.  Since {epsilon} is not
1178        // ambig with any real tokens, no error is reported
1179        // by deterministic() routines and we have to check
1180        // for the case where the lookahead depth didn't get
1181        // set to NONDETERMINISTIC (this only happens when the
1182        // FOLLOW contains real atoms + epsilon).
1183        boolean generateNonGreedyExitPath = false;
1184        int nonGreedyExitDepth = grammar.maxk;
1185
1186        if (!blk.greedy &&
1187            blk.exitLookaheadDepth <= grammar.maxk &&
1188            blk.exitCache[blk.exitLookaheadDepth].containsEpsilon()) {
1189            generateNonGreedyExitPath = true;
1190            nonGreedyExitDepth = blk.exitLookaheadDepth;
1191        }
1192        else if (!blk.greedy &&
1193            blk.exitLookaheadDepth == LLkGrammarAnalyzer.NONDETERMINISTIC) {
1194            generateNonGreedyExitPath = true;
1195        }
1196        if (generateNonGreedyExitPath) {
1197            if (DEBUG_CODE_GENERATOR) {
1198                System.out.println("nongreedy (...)* loop; exit depth is " +
1199                                   blk.exitLookaheadDepth);
1200            }
1201            String   predictExit =
1202                getLookaheadTestExpression(blk.exitCache,
1203                                           nonGreedyExitDepth);
1204            println("// nongreedy exit test");
1205            println("if (" + predictExit + ") break " + label + ";");
1206        }
1207
1208        JavaBlockFinishingInfo howToFinish = genCommonBlock(blk, false);
1209        genBlockFinish(howToFinish, "break " + label + ";");
1210
1211        tabs--;
1212        println("} while (true);");
1213        println("}");
1214
1215        // Restore previous AST generation
1216        currentASTResult = saveCurrentASTResult;
1217    }
1218
1219    /** Generate an alternative.
1220     * @param alt  The alternative to generate
1221     * @param blk The block to which the alternative belongs
1222     */
1223    protected void genAlt(Alternative alt, AlternativeBlock blk) {
1224        // Save the AST generation state, and set it to that of the alt
1225        boolean savegenAST = genAST;
1226        genAST = genAST && alt.getAutoGen();
1227
1228        boolean oldsaveTest = saveText;
1229        saveText = saveText && alt.getAutoGen();
1230
1231        // Reset the variable name map for the alternative
1232        Hashtable   saveMap = treeVariableMap;
1233        treeVariableMap = new Hashtable  ();
1234
1235        // Generate try block around the alt for  error handling
1236        if (alt.exceptionSpec != null) {
1237            println("try {      // for error handling");
1238            tabs++;
1239        }
1240
1241        AlternativeElement elem = alt.head;
1242        while (!(elem instanceof BlockEndElement)) {
1243            elem.generate(); // alt can begin with anything. Ask target to gen.
1244            elem = elem.next;
1245        }
1246
1247        if (genAST) {
1248            if (blk instanceof RuleBlock) {
1249                // Set the AST return value for the rule
1250                RuleBlock rblk = (RuleBlock)blk;
1251                if (grammar.hasSyntacticPredicate) {
1252                    // println("if ( inputState.guessing==0 ) {");
1253                    // tabs++;
1254                }
1255                println(rblk.getRuleName() + "_AST = (" + labeledElementASTType + ")currentAST.root;");
1256                if (grammar.hasSyntacticPredicate) {
1257                    // --tabs;
1258                    // println("}");
1259                }
1260            }
1261            else if (blk.getLabel() != null) {
1262                // ### future: also set AST value for labeled subrules.
1263                // println(blk.getLabel() + "_AST = ("+labeledElementASTType+")currentAST.root;");
1264                antlrTool.warning("Labeled subrules not yet supported", grammar.getFilename(), blk.getLine(), blk.getColumn());
1265            }
1266        }
1267
1268        if (alt.exceptionSpec != null) {
1269            // close try block
1270            tabs--;
1271            println("}");
1272            genErrorHandler(alt.exceptionSpec);
1273        }
1274
1275        genAST = savegenAST;
1276        saveText = oldsaveTest;
1277
1278        treeVariableMap = saveMap;
1279    }
1280
1281    /** Generate all the bitsets to be used in the parser or lexer
1282     * Generate the raw bitset data like "long _tokenSet1_data[] = {...};"
1283     * and the BitSet object declarations like "BitSet _tokenSet1 = new BitSet(_tokenSet1_data);"
1284     * Note that most languages do not support object initialization inside a
1285     * class definition, so other code-generators may have to separate the
1286     * bitset declarations from the initializations (e.g., put the initializations
1287     * in the generated constructor instead).
1288     * @param bitsetList The list of bitsets to generate.
1289     * @param maxVocabulary Ensure that each generated bitset can contain at least this value.
1290     */
1291    protected void genBitsets(Vector bitsetList,
1292                              int maxVocabulary
1293                              ) {
1294        println("");
1295        for (int i = 0; i < bitsetList.size(); i++) {
1296            BitSet p = (BitSet)bitsetList.elementAt(i);
1297            // Ensure that generated BitSet is large enough for vocabulary
1298            p.growToInclude(maxVocabulary);
1299            genBitSet(p, i);
1300        }
1301    }
1302
1303    /** Do something simple like:
1304     *  private static final long[] mk_tokenSet_0() {
1305     *    long[] data = { -2305839160922996736L, 63L, 16777216L, 0L, 0L, 0L };
1306     *    return data;
1307     *  }
1308     *  public static final BitSet _tokenSet_0 = new BitSet(mk_tokenSet_0());
1309     *
1310     *  Or, for large bitsets, optimize init so ranges are collapsed into loops.
1311     *  This is most useful for lexers using unicode.
1312     */
1313    private void genBitSet(BitSet p, int id) {
1314        // initialization data
1315        println(
1316            "private static final long[] mk" + getBitsetName(id) + "() {"
1317        );
1318        int n = p.lengthInLongWords();
1319        if ( n<BITSET_OPTIMIZE_INIT_THRESHOLD ) {
1320            println("\tlong[] data = { " + p.toStringOfWords() + "};");
1321        }
1322        else {
1323            // will init manually, allocate space then set values
1324            println("\tlong[] data = new long["+n+"];");
1325            long[] elems = p.toPackedArray();
1326            for (int i = 0; i < elems.length;) {
1327                if ( elems[i]==0 ) {
1328                    // done automatically by Java, don't waste time/code
1329                    i++;
1330                    continue;
1331                }
1332                if ( (i+1)==elems.length || elems[i]!=elems[i+1] ) {
1333                    // last number or no run of numbers, just dump assignment
1334                    println("\tdata["+i+"]="+elems[i]+"L;");
1335                    i++;
1336                }
1337                else {
1338                    // scan to find end of run
1339                    int j;
1340                    for (j = i + 1;
1341                         j < elems.length && elems[j]==elems[i];
1342                         j++)
1343                    {
1344                    }
1345                    // j-1 is last member of run
1346                    println("\tfor (int i = "+i+"; i<="+(j-1)+"; i++) { data[i]="+
1347                            elems[i]+"L; }");
1348                    i = j;
1349                }
1350            }
1351        }
1352
1353        println("\treturn data;");
1354        println("}");
1355        // BitSet object
1356        println(
1357            "public static final BitSet " + getBitsetName(id) + " = new BitSet(" +
1358            "mk" + getBitsetName(id) + "()" +
1359            ");"
1360        );
1361    }
1362
1363    /** Generate the finish of a block, using a combination of the info
1364     * returned from genCommonBlock() and the action to perform when
1365     * no alts were taken
1366     * @param howToFinish The return of genCommonBlock()
1367     * @param noViableAction What to generate when no alt is taken
1368     */
1369    private void genBlockFinish(JavaBlockFinishingInfo howToFinish, String   noViableAction) {
1370        if (howToFinish.needAnErrorClause &&
1371            (howToFinish.generatedAnIf || howToFinish.generatedSwitch)) {
1372            if (howToFinish.generatedAnIf) {
1373                println("else {");
1374            }
1375            else {
1376                println("{");
1377            }
1378            tabs++;
1379            println(noViableAction);
1380            tabs--;
1381            println("}");
1382        }
1383
1384        if (howToFinish.postscript != null) {
1385            println(howToFinish.postscript);
1386        }
1387    }
1388
1389    /** Generate the init action for a block, which may be a RuleBlock or a
1390     * plain AlternativeBLock.
1391     * @blk The block for which the preamble is to be generated.
1392     */
1393    protected void genBlockInitAction(AlternativeBlock blk) {
1394        // dump out init action
1395        if (blk.initAction != null) {
1396            printAction(processActionForSpecialSymbols(blk.initAction, blk.getLine(), currentRule, null));
1397        }
1398    }
1399
1400    /** Generate the header for a block, which may be a RuleBlock or a
1401     * plain AlternativeBLock.  This generates any variable declarations
1402     * and syntactic-predicate-testing variables.
1403     * @blk The block for which the preamble is to be generated.
1404     */
1405    protected void genBlockPreamble(AlternativeBlock blk) {
1406        // define labels for rule blocks.
1407        if (blk instanceof RuleBlock) {
1408            RuleBlock rblk = (RuleBlock)blk;
1409            if (rblk.labeledElements != null) {
1410                for (int i = 0; i < rblk.labeledElements.size(); i++) {
1411                    AlternativeElement a = (AlternativeElement)rblk.labeledElements.elementAt(i);
1412                    // System.out.println("looking at labeled element: "+a);
1413                    // Variables for labeled rule refs and
1414                    // subrules are different than variables for
1415                    // grammar atoms.  This test is a little tricky
1416                    // because we want to get all rule refs and ebnf,
1417                    // but not rule blocks or syntactic predicates
1418                    if (
1419                        a instanceof RuleRefElement ||
1420                        a instanceof AlternativeBlock &&
1421                        !(a instanceof RuleBlock) &&
1422                        !(a instanceof SynPredBlock)
1423                    ) {
1424
1425                        if (
1426                            !(a instanceof RuleRefElement) &&
1427                            ((AlternativeBlock)a).not &&
1428                            analyzer.subruleCanBeInverted(((AlternativeBlock)a), grammar instanceof LexerGrammar)
1429                        ) {
1430                            // Special case for inverted subrules that
1431                            // will be inlined.  Treat these like
1432                            // token or char literal references
1433                            println(labeledElementType + " " + a.getLabel() + " = " + labeledElementInit + ";");
1434                            if (grammar.buildAST) {
1435                                genASTDeclaration(a);
1436                            }
1437                        }
1438                        else {
1439                            if (grammar.buildAST) {
1440                                // Always gen AST variables for
1441                                // labeled elements, even if the
1442                                // element itself is marked with !
1443                                genASTDeclaration(a);
1444                            }
1445                            if (grammar instanceof LexerGrammar) {
1446                                println("Token " + a.getLabel() + "=null;");
1447                            }
1448                            if (grammar instanceof TreeWalkerGrammar) {
1449                                // always generate rule-ref variables
1450                                // for tree walker
1451                                println(labeledElementType + " " + a.getLabel() + " = " + labeledElementInit + ";");
1452                            }
1453                        }
1454                    }
1455                    else {
1456                        // It is a token or literal reference.  Generate the
1457                        // correct variable type for this grammar
1458                        println(labeledElementType + " " + a.getLabel() + " = " + labeledElementInit + ";");
1459
1460                        // In addition, generate *_AST variables if
1461                        // building ASTs
1462                        if (grammar.buildAST) {
1463                            if (a instanceof GrammarAtom &&
1464                                ((GrammarAtom)a).getASTNodeType() != null) {
1465                                GrammarAtom ga = (GrammarAtom)a;
1466                                genASTDeclaration(a, ga.getASTNodeType());
1467                            }
1468                            else {
1469                                genASTDeclaration(a);
1470                            }
1471                        }
1472                    }
1473                }
1474            }
1475        }
1476    }
1477
1478    /** Generate a series of case statements that implement a BitSet test.
1479     * @param p The Bitset for which cases are to be generated
1480     */
1481    protected void genCases(BitSet p) {
1482        if (DEBUG_CODE_GENERATOR) System.out.println("genCases(" + p + ")");
1483        int[] elems;
1484
1485        elems = p.toArray();
1486        // Wrap cases four-per-line for lexer, one-per-line for parser
1487        int wrap = (grammar instanceof LexerGrammar) ? 4 : 1;
1488        int j = 1;
1489        boolean startOfLine = true;
1490        for (int i = 0; i < elems.length; i++) {
1491            if (j == 1) {
1492                print("");
1493            }
1494            else {
1495                _print("  ");
1496            }
1497            _print("case " + getValueString(elems[i]) + ":");
1498
1499            if (j == wrap) {
1500                _println("");
1501                startOfLine = true;
1502                j = 1;
1503            }
1504            else {
1505                j++;
1506                startOfLine = false;
1507            }
1508        }
1509        if (!startOfLine) {
1510            _println("");
1511        }
1512    }
1513
1514    /**Generate common code for a block of alternatives; return a
1515     * postscript that needs to be generated at the end of the
1516     * block.  Other routines may append else-clauses and such for
1517     * error checking before the postfix is generated.  If the
1518     * grammar is a lexer, then generate alternatives in an order
1519     * where alternatives requiring deeper lookahead are generated
1520     * first, and EOF in the lookahead set reduces the depth of
1521     * the lookahead.  @param blk The block to generate @param
1522     * noTestForSingle If true, then it does not generate a test
1523     * for a single alternative.
1524     */
1525    public JavaBlockFinishingInfo genCommonBlock(AlternativeBlock blk,
1526                                                 boolean noTestForSingle) {
1527        int nIF = 0;
1528        boolean createdLL1Switch = false;
1529        int closingBracesOfIFSequence = 0;
1530        JavaBlockFinishingInfo finishingInfo = new JavaBlockFinishingInfo();
1531        if (DEBUG_CODE_GENERATOR) System.out.println("genCommonBlock(" + blk + ")");
1532
1533        // Save the AST generation state, and set it to that of the block
1534        boolean savegenAST = genAST;
1535        genAST = genAST && blk.getAutoGen();
1536
1537        boolean oldsaveTest = saveText;
1538        saveText = saveText && blk.getAutoGen();
1539
1540        // Is this block inverted?  If so, generate special-case code
1541        if (
1542            blk.not &&
1543            analyzer.subruleCanBeInverted(blk, grammar instanceof LexerGrammar)
1544        ) {
1545            if (DEBUG_CODE_GENERATOR) System.out.println("special case: ~(subrule)");
1546            Lookahead p = analyzer.look(1, blk);
1547            // Variable assignment for labeled elements
1548            if (blk.getLabel() != null && syntacticPredLevel == 0) {
1549                println(blk.getLabel() + " = " + lt1Value + ";");
1550            }
1551
1552            // AST
1553            genElementAST(blk);
1554
1555            String   astArgs = "";
1556            if (grammar instanceof TreeWalkerGrammar) {
1557                astArgs = "_t,";
1558            }
1559
1560            // match the bitset for the alternative
1561            println("match(" + astArgs + getBitsetName(markBitsetForGen(p.fset)) + ");");
1562
1563            // tack on tree cursor motion if doing a tree walker
1564            if (grammar instanceof TreeWalkerGrammar) {
1565                println("_t = _t.getNextSibling();");
1566            }
1567            return finishingInfo;
1568        }
1569
1570        // Special handling for single alt
1571        if (blk.getAlternatives().size() == 1) {
1572            Alternative alt = blk.getAlternativeAt(0);
1573            // Generate a warning if there is a synPred for single alt.
1574            if (alt.synPred != null) {
1575                antlrTool.warning(
1576                    "Syntactic predicate superfluous for single alternative",
1577                    grammar.getFilename(),
1578                    blk.getAlternativeAt(0).synPred.getLine(),
1579                    blk.getAlternativeAt(0).synPred.getColumn()
1580                );
1581            }
1582            if (noTestForSingle) {
1583                if (alt.semPred != null) {
1584                    // Generate validating predicate
1585                    genSemPred(alt.semPred, blk.line);
1586                }
1587                genAlt(alt, blk);
1588                return finishingInfo;
1589            }
1590        }
1591
1592        // count number of simple LL(1) cases; only do switch for
1593        // many LL(1) cases (no preds, no end of token refs)
1594        // We don't care about exit paths for (...)*, (...)+
1595        // because we don't explicitly have a test for them
1596        // as an alt in the loop.
1597        //
1598        // Also, we now count how many unicode lookahead sets
1599        // there are--they must be moved to DEFAULT or ELSE
1600        // clause.
1601        int nLL1 = 0;
1602        for (int i = 0; i < blk.getAlternatives().size(); i++) {
1603            Alternative a = blk.getAlternativeAt(i);
1604            if (suitableForCaseExpression(a)) {
1605                nLL1++;
1606            }
1607        }
1608
1609        // do LL(1) cases
1610        if (nLL1 >= makeSwitchThreshold) {
1611            // Determine the name of the item to be compared
1612            String   testExpr = lookaheadString(1);
1613            createdLL1Switch = true;
1614            // when parsing trees, convert null to valid tree node with NULL lookahead
1615            if (grammar instanceof TreeWalkerGrammar) {
1616                println("if (_t==null) _t=ASTNULL;");
1617            }
1618            println("switch ( " + testExpr + ") {");
1619            for (int i = 0; i < blk.alternatives.size(); i++) {
1620                Alternative alt = blk.getAlternativeAt(i);
1621                // ignore any non-LL(1) alts, predicated alts,
1622                // or end-of-token alts for case expressions
1623                if (!suitableForCaseExpression(alt)) {
1624                    continue;
1625                }
1626                Lookahead p = alt.cache[1];
1627                if (p.fset.degree() == 0 && !p.containsEpsilon()) {
1628                    antlrTool.warning("Alternate omitted due to empty prediction set",
1629                                 grammar.getFilename(),
1630                                 alt.head.getLine(), alt.head.getColumn());
1631                }
1632                else {
1633                    genCases(p.fset);
1634                    println("{");
1635                    tabs++;
1636                    genAlt(alt, blk);
1637                    println("break;");
1638                    tabs--;
1639                    println("}");
1640                }
1641            }
1642            println("default:");
1643            tabs++;
1644        }
1645
1646        // do non-LL(1) and nondeterministic cases This is tricky in
1647        // the lexer, because of cases like: STAR : '*' ; ASSIGN_STAR
1648        // : "*="; Since nextToken is generated without a loop, then
1649        // the STAR will have end-of-token as it's lookahead set for
1650        // LA(2).  So, we must generate the alternatives containing
1651        // trailing end-of-token in their lookahead sets *after* the
1652        // alternatives without end-of-token.  This implements the
1653        // usual lexer convention that longer matches come before
1654        // shorter ones, e.g.  "*=" matches ASSIGN_STAR not STAR
1655        //
1656        // For non-lexer grammars, this does not sort the alternates
1657        // by depth Note that alts whose lookahead is purely
1658        // end-of-token at k=1 end up as default or else clauses.
1659        int startDepth = (grammar instanceof LexerGrammar) ? grammar.maxk : 0;
1660        for (int altDepth = startDepth; altDepth >= 0; altDepth--) {
1661            if (DEBUG_CODE_GENERATOR) System.out.println("checking depth " + altDepth);
1662            for (int i = 0; i < blk.alternatives.size(); i++) {
1663                Alternative alt = blk.getAlternativeAt(i);
1664                if (DEBUG_CODE_GENERATOR) System.out.println("genAlt: " + i);
1665                // if we made a switch above, ignore what we already took care
1666                // of.  Specifically, LL(1) alts with no preds
1667                // that do not have end-of-token in their prediction set
1668                // and that are not giant unicode sets.
1669                if (createdLL1Switch && suitableForCaseExpression(alt)) {
1670                    if (DEBUG_CODE_GENERATOR) System.out.println("ignoring alt because it was in the switch");
1671                    continue;
1672                }
1673                String   e;
1674
1675                boolean unpredicted = false;
1676
1677                if (grammar instanceof LexerGrammar) {
1678                    // Calculate the "effective depth" of the alt,
1679                    // which is the max depth at which
1680                    // cache[depth]!=end-of-token
1681                    int effectiveDepth = alt.lookaheadDepth;
1682                    if (effectiveDepth == GrammarAnalyzer.NONDETERMINISTIC) {
1683                        // use maximum lookahead
1684                        effectiveDepth = grammar.maxk;
1685                    }
1686                    while (effectiveDepth >= 1 &&
1687                        alt.cache[effectiveDepth].containsEpsilon()) {
1688                        effectiveDepth--;
1689                    }
1690                    // Ignore alts whose effective depth is other than
1691                    // the ones we are generating for this iteration.
1692                    if (effectiveDepth != altDepth) {
1693                        if (DEBUG_CODE_GENERATOR)
1694                            System.out.println("ignoring alt because effectiveDepth!=altDepth;" + effectiveDepth + "!=" + altDepth);
1695                        continue;
1696                    }
1697                    unpredicted = lookaheadIsEmpty(alt, effectiveDepth);
1698                    e = getLookaheadTestExpression(alt, effectiveDepth);
1699                }
1700                else {
1701                    unpredicted = lookaheadIsEmpty(alt, grammar.maxk);
1702                    e = getLookaheadTestExpression(alt, grammar.maxk);
1703                }
1704
1705                // Was it a big unicode range that forced unsuitability
1706                // for a case expression?
1707                if (alt.cache[1].fset.degree() > caseSizeThreshold &&
1708                    suitableForCaseExpression(alt)) {
1709                    if (nIF == 0) {
1710                        println("if " + e + " {");
1711                    }
1712                    else {
1713                        println("else if " + e + " {");
1714                    }
1715                }
1716                else if (unpredicted &&
1717                    alt.semPred == null &&
1718                    alt.synPred == null) {
1719                    // The alt has empty prediction set and no
1720                    // predicate to help out.  if we have not
1721                    // generated a previous if, just put {...} around
1722                    // the end-of-token clause
1723                    if (nIF == 0) {
1724                        println("{");
1725                    }
1726                    else {
1727                        println("else {");
1728                    }
1729                    finishingInfo.needAnErrorClause = false;
1730                }
1731                else { // check for sem and syn preds
1732
1733                    // Add any semantic predicate expression to the
1734                    // lookahead test
1735                    if (alt.semPred != null) {
1736                        // if debugging, wrap the evaluation of the
1737                        // predicate in a method translate $ and #
1738                        // references
1739                        ActionTransInfo tInfo = new ActionTransInfo();
1740                        String   actionStr =
1741                            processActionForSpecialSymbols(alt.semPred,
1742                                                           blk.line,
1743                                                           currentRule,
1744                                                           tInfo);
1745                        // ignore translation info...we don't need to
1746                        // do anything with it.  call that will inform
1747                        // SemanticPredicateListeners of the result
1748                        if (((grammar instanceof ParserGrammar) ||
1749                            (grammar instanceof LexerGrammar)) &&
1750                            grammar.debuggingOutput) {
1751                            e = "(" + e + "&& fireSemanticPredicateEvaluated(persistence.antlr.debug.SemanticPredicateEvent.PREDICTING," +
1752                                addSemPred(charFormatter.escapeString(actionStr)) + "," + actionStr + "))";
1753                        }
1754                        else {
1755                            e = "(" + e + "&&(" + actionStr + "))";
1756                        }
1757                    }
1758
1759                    // Generate any syntactic predicates
1760                    if (nIF > 0) {
1761                        if (alt.synPred != null) {
1762                            println("else {");
1763                            tabs++;
1764                            genSynPred(alt.synPred, e);
1765                            closingBracesOfIFSequence++;
1766                        }
1767                        else {
1768                            println("else if " + e + " {");
1769                        }
1770                    }
1771                    else {
1772                        if (alt.synPred != null) {
1773                            genSynPred(alt.synPred, e);
1774                        }
1775                        else {
1776                            // when parsing trees, convert null to
1777                            // valid tree node with NULL lookahead.
1778                            if (grammar instanceof TreeWalkerGrammar) {
1779                                println("if (_t==null) _t=ASTNULL;");
1780                            }
1781                            println("if " + e + " {");
1782                        }
1783                    }
1784
1785                }
1786
1787                nIF++;
1788                tabs++;
1789                genAlt(alt, blk);
1790                tabs--;
1791                println("}");
1792            }
1793        }
1794        String   ps = "";
1795        for (int i = 1; i <= closingBracesOfIFSequence; i++) {
1796            ps += "}";
1797        }
1798
1799        // Restore the AST generation state
1800        genAST = savegenAST;
1801
1802        // restore save text state
1803        saveText = oldsaveTest;
1804
1805        // Return the finishing info.
1806        if (createdLL1Switch) {
1807            tabs--;
1808            finishingInfo.postscript = ps + "}";
1809            finishingInfo.generatedSwitch = true;
1810            finishingInfo.generatedAnIf = nIF > 0;
1811            //return new JavaBlockFinishingInfo(ps+"}",true,nIF>0); // close up switch statement
1812
1813        }
1814        else {
1815            finishingInfo.postscript = ps;
1816            finishingInfo.generatedSwitch = false;
1817            finishingInfo.generatedAnIf = nIF > 0;
1818            // return new JavaBlockFinishingInfo(ps, false,nIF>0);
1819        }
1820        return finishingInfo;
1821    }
1822
1823    private static boolean suitableForCaseExpression(Alternative a) {
1824        return
1825            a.lookaheadDepth == 1 &&
1826            a.semPred == null &&
1827            !a.cache[1].containsEpsilon() &&
1828            a.cache[1].fset.degree() <= caseSizeThreshold;
1829    }
1830
1831    /** Generate code to link an element reference into the AST */
1832    private void genElementAST(AlternativeElement el) {
1833        // handle case where you're not building trees, but are in tree walker.
1834        // Just need to get labels set up.
1835        if (grammar instanceof TreeWalkerGrammar && !grammar.buildAST) {
1836            String   elementRef;
1837            String   astName;
1838
1839            // Generate names and declarations of the AST variable(s)
1840            if (el.getLabel() == null) {
1841                elementRef = lt1Value;
1842                // Generate AST variables for unlabeled stuff
1843                astName = "tmp" + astVarNumber + "_AST";
1844                astVarNumber++;
1845                // Map the generated AST variable in the alternate
1846                mapTreeVariable(el, astName);
1847                // Generate an "input" AST variable also
1848                println(labeledElementASTType + " " + astName + "_in = " + elementRef + ";");
1849            }
1850            return;
1851        }
1852
1853        if (grammar.buildAST && syntacticPredLevel == 0) {
1854            boolean needASTDecl =
1855                (genAST &&
1856                (el.getLabel() != null ||
1857                el.getAutoGenType() != GrammarElement.AUTO_GEN_BANG
1858                )
1859                );
1860
1861            // RK: if we have a grammar element always generate the decl
1862            // since some guy can access it from an action and we can't
1863            // peek ahead (well not without making a mess).
1864            // I'd prefer taking this out.
1865            if (el.getAutoGenType() != GrammarElement.AUTO_GEN_BANG &&
1866                (el instanceof TokenRefElement))
1867            {
1868                needASTDecl = true;
1869            }
1870
1871            boolean doNoGuessTest =
1872                (grammar.hasSyntacticPredicate && needASTDecl);
1873
1874            String   elementRef;
1875            String   astNameBase;
1876
1877            // Generate names and declarations of the AST variable(s)
1878            if (el.getLabel() != null) {
1879                elementRef = el.getLabel();
1880                astNameBase = el.getLabel();
1881            }
1882            else {
1883                elementRef = lt1Value;
1884                // Generate AST variables for unlabeled stuff
1885                astNameBase = "tmp" + astVarNumber;
1886                ;
1887                astVarNumber++;
1888            }
1889
1890            // Generate the declaration if required.
1891            if (needASTDecl) {
1892                // Generate the declaration
1893                if (el instanceof GrammarAtom) {
1894                    GrammarAtom ga = (GrammarAtom)el;
1895                    if (ga.getASTNodeType() != null) {
1896                        genASTDeclaration(el, astNameBase, ga.getASTNodeType());
1897//                      println(ga.getASTNodeType()+" " + astName+" = null;");
1898                    }
1899                    else {
1900                        genASTDeclaration(el, astNameBase, labeledElementASTType);
1901//                      println(labeledElementASTType+" " + astName + " = null;");
1902                    }
1903                }
1904                else {
1905                    genASTDeclaration(el, astNameBase, labeledElementASTType);
1906//                  println(labeledElementASTType+" " + astName + " = null;");
1907                }
1908            }
1909
1910            // for convenience..
1911            String   astName = astNameBase + "_AST";
1912
1913            // Map the generated AST variable in the alternate
1914            mapTreeVariable(el, astName);
1915            if (grammar instanceof TreeWalkerGrammar) {
1916                // Generate an "input" AST variable also
1917                println(labeledElementASTType + " " + astName + "_in = null;");
1918            }
1919
1920            // Enclose actions with !guessing
1921            if (doNoGuessTest) {
1922                // println("if (inputState.guessing==0) {");
1923                // tabs++;
1924            }
1925
1926            // if something has a label assume it will be used
1927            // so we must initialize the RefAST
1928            if (el.getLabel() != null) {
1929                if (el instanceof GrammarAtom) {
1930                    println(astName + " = " + getASTCreateString((GrammarAtom)el, elementRef) + ";");
1931                }
1932                else {
1933                    println(astName + " = " + getASTCreateString(elementRef) + ";");
1934                }
1935            }
1936
1937            // if it has no label but a declaration exists initialize it.
1938            if (el.getLabel() == null && needASTDecl) {
1939                elementRef = lt1Value;
1940                if (el instanceof GrammarAtom) {
1941                    println(astName + " = " + getASTCreateString((GrammarAtom)el, elementRef) + ";");
1942                }
1943                else {
1944                    println(astName + " = " + getASTCreateString(elementRef) + ";");
1945                }
1946                // Map the generated AST variable in the alternate
1947                if (grammar instanceof TreeWalkerGrammar) {
1948                    // set "input" AST variable also
1949                    println(astName + "_in = " + elementRef + ";");
1950                }
1951            }
1952
1953            if (genAST) {
1954                switch (el.getAutoGenType()) {
1955                    case GrammarElement.AUTO_GEN_NONE:
1956                        println("astFactory.addASTChild(currentAST, " + astName + ");");
1957                        break;
1958                    case GrammarElement.AUTO_GEN_CARET:
1959                        println("astFactory.makeASTRoot(currentAST, " + astName + ");");
1960                        break;
1961                    default:
1962                        break;
1963                }
1964            }
1965            if (doNoGuessTest) {
1966                // tabs--;
1967                // println("}");
1968            }
1969        }
1970    }
1971
1972    /** Close the try block and generate catch phrases
1973     * if the element has a labeled handler in the rule
1974     */
1975    private void genErrorCatchForElement(AlternativeElement el) {
1976        if (el.getLabel() == null) return;
1977        String   r = el.enclosingRuleName;
1978        if (grammar instanceof LexerGrammar) {
1979            r = CodeGenerator.encodeLexerRuleName(el.enclosingRuleName);
1980        }
1981        RuleSymbol rs = (RuleSymbol)grammar.getSymbol(r);
1982        if (rs == null) {
1983            antlrTool.panic("Enclosing rule not found!");
1984        }
1985        ExceptionSpec ex = rs.block.findExceptionSpec(el.getLabel());
1986        if (ex != null) {
1987            tabs--;
1988            println("}");
1989            genErrorHandler(ex);
1990        }
1991    }
1992
1993    /** Generate the catch phrases for a user-specified error handler */
1994    private void genErrorHandler(ExceptionSpec ex) {
1995        // Each ExceptionHandler in the ExceptionSpec is a separate catch
1996        for (int i = 0; i < ex.handlers.size(); i++) {
1997            ExceptionHandler handler = (ExceptionHandler)ex.handlers.elementAt(i);
1998            // Generate catch phrase
1999            println("catch (" + handler.exceptionTypeAndName.getText() + ") {");
2000            tabs++;
2001            if (grammar.hasSyntacticPredicate) {
2002                println("if (inputState.guessing==0) {");
2003                tabs++;
2004            }
2005
2006            // When not guessing, execute user handler action
2007            ActionTransInfo tInfo = new ActionTransInfo();
2008            printAction(
2009                processActionForSpecialSymbols(handler.action.getText(),
2010                                               handler.action.getLine(),
2011                                               currentRule, tInfo)
2012            );
2013
2014            if (grammar.hasSyntacticPredicate) {
2015                tabs--;
2016                println("} else {");
2017                tabs++;
2018                // When guessing, rethrow exception
2019                println(
2020                    "throw " +
2021                    extractIdOfAction(handler.exceptionTypeAndName) +
2022                    ";"
2023                );
2024                tabs--;
2025                println("}");
2026            }
2027            // Close catch phrase
2028            tabs--;
2029            println("}");
2030        }
2031    }
2032
2033    /** Generate a try { opening if the element has a labeled handler in the rule */
2034    private void genErrorTryForElement(AlternativeElement el) {
2035        if (el.getLabel() == null) return;
2036        String   r = el.enclosingRuleName;
2037        if (grammar instanceof LexerGrammar) {
2038            r = CodeGenerator.encodeLexerRuleName(el.enclosingRuleName);
2039        }
2040        RuleSymbol rs = (RuleSymbol)grammar.getSymbol(r);
2041        if (rs == null) {
2042            antlrTool.panic("Enclosing rule not found!");
2043        }
2044        ExceptionSpec ex = rs.block.findExceptionSpec(el.getLabel());
2045        if (ex != null) {
2046            println("try { // for error handling");
2047            tabs++;
2048        }
2049    }
2050
2051    protected void genASTDeclaration(AlternativeElement el) {
2052        genASTDeclaration(el, labeledElementASTType);
2053    }
2054
2055    protected void genASTDeclaration(AlternativeElement el, String   node_type) {
2056        genASTDeclaration(el, el.getLabel(), node_type);
2057    }
2058
2059    protected void genASTDeclaration(AlternativeElement el, String   var_name, String   node_type) {
2060        // already declared?
2061        if (declaredASTVariables.contains(el))
2062            return;
2063
2064        // emit code
2065        println(node_type + " " + var_name + "_AST = null;");
2066
2067        // mark as declared
2068        declaredASTVariables.put(el,el);
2069    }
2070
2071    /** Generate a header that is common to all Java files */
2072    protected void genHeader() {
2073        println("// $ANTLR " + Tool.version + ": " +
2074                "\"" + antlrTool.fileMinusPath(antlrTool.grammarFile) + "\"" +
2075                " -> " +
2076                "\"" + grammar.getClassName() + ".java\"$");
2077    }
2078
2079    private void genLiteralsTest() {
2080        println("_ttype = testLiteralsTable(_ttype);");
2081    }
2082
2083    private void genLiteralsTestForPartialToken() {
2084        println("_ttype = testLiteralsTable(new String(text.getBuffer(),_begin,text.length()-_begin),_ttype);");
2085    }
2086
2087    protected void genMatch(BitSet b) {
2088    }
2089
2090    protected void genMatch(GrammarAtom atom) {
2091        if (atom instanceof StringLiteralElement) {
2092            if (grammar instanceof LexerGrammar) {
2093                genMatchUsingAtomText(atom);
2094            }
2095            else {
2096                genMatchUsingAtomTokenType(atom);
2097            }
2098        }
2099        else if (atom instanceof CharLiteralElement) {
2100            if (grammar instanceof LexerGrammar) {
2101                genMatchUsingAtomText(atom);
2102            }
2103            else {
2104                antlrTool.error("cannot ref character literals in grammar: " + atom);
2105            }
2106        }
2107        else if (atom instanceof TokenRefElement) {
2108            genMatchUsingAtomText(atom);
2109        }
2110        else if (atom instanceof WildcardElement) {
2111            gen((WildcardElement)atom);
2112        }
2113    }
2114
2115    protected void genMatchUsingAtomText(GrammarAtom atom) {
2116        // match() for trees needs the _t cursor
2117        String   astArgs = "";
2118        if (grammar instanceof TreeWalkerGrammar) {
2119            astArgs = "_t,";
2120        }
2121
2122        // if in lexer and ! on element, save buffer index to kill later
2123        if (grammar instanceof LexerGrammar && (!saveText || atom.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
2124            println("_saveIndex=text.length();");
2125        }
2126
2127        print(atom.not ? "matchNot(" : "match(");
2128        _print(astArgs);
2129
2130        // print out what to match
2131        if (atom.atomText.equals("EOF")) {
2132            // horrible hack to handle EOF case
2133            _print("Token.EOF_TYPE");
2134        }
2135        else {
2136            _print(atom.atomText);
2137        }
2138        _println(");");
2139
2140        if (grammar instanceof LexerGrammar && (!saveText || atom.getAutoGenType() == GrammarElement.AUTO_GEN_BANG)) {
2141            println("text.setLength(_saveIndex);");     // kill text atom put in buffer
2142        }
2143    }
2144
2145    protected void genMatchUsingAtomTokenType(GrammarAtom atom) {
2146        // match() for trees needs the _t cursor
2147        String   astArgs = "";
2148        if (grammar instanceof TreeWalkerGrammar) {
2149            astArgs = "_t,";
2150        }
2151
2152        // If the literal can be mangled, generate the symbolic constant instead
2153        String   mangledName = null;
2154        String   s = astArgs + getValueString(atom.getType());
2155
2156        // matching
2157        println((atom.not ? "matchNot(" : "match(") + s + ");");
2158    }
2159
2160    /** Generate the nextToken() rule.  nextToken() is a synthetic
2161     * lexer rule that is the implicit OR of all user-defined
2162     * lexer rules.
2163     */
2164    public void genNextToken() {
2165        // Are there any public rules?  If not, then just generate a
2166        // fake nextToken().
2167        boolean hasPublicRules = false;
2168        for (int i = 0; i < grammar.rules.size(); i++) {
2169            RuleSymbol rs = (RuleSymbol)grammar.rules.elementAt(i);
2170            if (rs.isDefined() && rs.access.equals("public")) {
2171                hasPublicRules = true;
2172                break;
2173            }
2174        }
2175        if (!hasPublicRules) {
2176            println("");
2177            println("public Token nextToken() throws TokenStreamException {");
2178            println("\ttry {uponEOF();}");
2179            println("\tcatch(CharStreamIOException csioe) {");
2180            println("\t\tthrow new TokenStreamIOException(csioe.io);");
2181            println("\t}");
2182            println("\tcatch(CharStreamException cse) {");
2183            println("\t\tthrow new TokenStreamException(cse.getMessage());");
2184            println("\t}");
2185            println("\treturn new CommonToken(Token.EOF_TYPE, \"\");");
2186            println("}");
2187            println("");
2188            return;
2189        }
2190
2191        // Create the synthesized nextToken() rule
2192        RuleBlock nextTokenBlk = MakeGrammar.createNextTokenRule(grammar, grammar.rules, "nextToken");
2193        // Define the nextToken rule symbol
2194        RuleSymbol nextTokenRs = new RuleSymbol("mnextToken");
2195        nextTokenRs.setDefined();
2196        nextTokenRs.setBlock(nextTokenBlk);
2197        nextTokenRs.access = "private";
2198        grammar.define(nextTokenRs);
2199        // Analyze the nextToken rule
2200        boolean ok = grammar.theLLkAnalyzer.deterministic(nextTokenBlk);
2201
2202        // Generate the next token rule
2203        String   filterRule = null;
2204        if (((LexerGrammar)grammar).filterMode) {
2205            filterRule = ((LexerGrammar)grammar).filterRule;
2206        }
2207
2208        println("");
2209        println("public Token nextToken() throws TokenStreamException {");
2210        tabs++;
2211        println("Token theRetToken=null;");
2212        _println("tryAgain:");
2213        println("for (;;) {");
2214        tabs++;
2215        println("Token _token = null;");
2216        println("int _ttype = Token.INVALID_TYPE;");
2217        if (((LexerGrammar)grammar).filterMode) {
2218            println("setCommitToPath(false);");
2219            if (filterRule != null) {
2220                // Here's a good place to ensure that the filter rule actually exists
2221                if (!grammar.isDefined(CodeGenerator.encodeLexerRuleName(filterRule))) {
2222                    grammar.antlrTool.error("Filter rule " + filterRule + " does not exist in this lexer");
2223                }
2224                else {
2225                    RuleSymbol rs = (RuleSymbol)grammar.getSymbol(CodeGenerator.encodeLexerRuleName(filterRule));
2226                    if (!rs.isDefined()) {
2227                        grammar.antlrTool.error("Filter rule " + filterRule + " does not exist in this lexer");
2228                    }
2229                    else if (rs.access.equals("public")) {
2230                        grammar.antlrTool.error("Filter rule " + filterRule + " must be protected");
2231                    }
2232                }
2233                println("int _m;");
2234                println("_m = mark();");
2235            }
2236        }
2237        println("resetText();");
2238
2239        println("try {   // for char stream error handling");
2240        tabs++;
2241
2242        // Generate try around whole thing to trap scanner errors
2243        println("try {   // for lexical error handling");
2244        tabs++;
2245
2246        // Test for public lexical rules with empty paths
2247        for (int i = 0; i < nextTokenBlk.getAlternatives().size(); i++) {
2248            Alternative a = nextTokenBlk.getAlternativeAt(i);
2249            if (a.cache[1].containsEpsilon()) {
2250                //String r = a.head.toString();
2251                RuleRefElement rr = (RuleRefElement)a.head;
2252                String   r = CodeGenerator.decodeLexerRuleName(rr.targetRule);
2253                antlrTool.warning("public lexical rule "+r+" is optional (can match \"nothing\")");
2254            }
2255        }
2256
2257        // Generate the block
2258        String   newline = System.getProperty("line.separator");
2259        JavaBlockFinishingInfo howToFinish = genCommonBlock(nextTokenBlk, false);
2260        String   errFinish = "if (LA(1)==EOF_CHAR) {uponEOF(); _returnToken = makeToken(Token.EOF_TYPE);}";
2261        errFinish += newline + "\t\t\t\t";
2262        if (((LexerGrammar)grammar).filterMode) {
2263            if (filterRule == null) {
2264                errFinish += "else {consume(); continue tryAgain;}";
2265            }
2266            else {
2267                errFinish += "else {" + newline +
2268                    "\t\t\t\t\tcommit();" + newline +
2269                    "\t\t\t\t\ttry {m" + filterRule + "(false);}" + newline +
2270                    "\t\t\t\t\tcatch(RecognitionException e) {" + newline +
2271                    "\t\t\t\t\t // catastrophic failure" + newline +
2272                    "\t\t\t\t\t reportError(e);" + newline +
2273                    "\t\t\t\t\t consume();" + newline +
2274                    "\t\t\t\t\t}" + newline +
2275                    "\t\t\t\t\tcontinue tryAgain;" + newline +
2276                    "\t\t\t\t}";
2277            }
2278        }
2279        else {
2280            errFinish += "else {" + throwNoViable + "}";
2281        }
2282        genBlockFinish(howToFinish, errFinish);
2283
2284        // at this point a valid token has been matched, undo "mark" that was done
2285        if (((LexerGrammar)grammar).filterMode && filterRule != null) {
2286            println("commit();");
2287        }
2288
2289        // Generate literals test if desired
2290        // make sure _ttype is set first; note _returnToken must be
2291        // non-null as the rule was required to create it.
2292        println("if ( _returnToken==null ) continue tryAgain; // found SKIP token");
2293        println("_ttype = _returnToken.getType();");
2294        if (((LexerGrammar)grammar).getTestLiterals()) {
2295            genLiteralsTest();
2296        }
2297
2298        // return token created by rule reference in switch
2299        println("_returnToken.setType(_ttype);");
2300        println("return _returnToken;");
2301
2302        // Close try block
2303        tabs--;
2304        println("}");
2305        println("catch (RecognitionException e) {");
2306        tabs++;
2307        if (((LexerGrammar)grammar).filterMode) {
2308            if (filterRule == null) {
2309                println("if ( !getCommitToPath() ) {consume(); continue tryAgain;}");
2310            }
2311            else {
2312                println("if ( !getCommitToPath() ) {");
2313                tabs++;
2314                println("rewind(_m);");
2315                println("resetText();");
2316                println("try {m" + filterRule + "(false);}");
2317                println("catch(RecognitionException ee) {");
2318                println("   // horrendous failure: error in filter rule");
2319                println("   reportError(ee);");
2320                println("   consume();");
2321                println("}");
2322                println("continue tryAgain;");
2323                tabs--;
2324                println("}");
2325            }
2326        }
2327        if (nextTokenBlk.getDefaultErrorHandler()) {
2328            println("reportError(e);");
2329            println("consume();");
2330        }
2331        else {
2332            // pass on to invoking routine
2333            println("throw new TokenStreamRecognitionException(e);");
2334        }
2335        tabs--;
2336        println("}");
2337
2338        // close CharStreamException try
2339        tabs--;
2340        println("}");
2341        println("catch (CharStreamException cse) {");
2342        println("   if ( cse instanceof CharStreamIOException ) {");
2343        println("       throw new TokenStreamIOException(((CharStreamIOException)cse).io);");
2344        println("   }");
2345        println("   else {");
2346        println("       throw new TokenStreamException(cse.getMessage());");
2347        println("   }");
2348        println("}");
2349
2350        // close for-loop
2351        tabs--;
2352        println("}");
2353
2354        // close method nextToken
2355        tabs--;
2356        println("}");
2357        println("");
2358    }
2359
2360    /** Gen a named rule block.
2361     * ASTs are generated for each element of an alternative unless
2362     * the rule or the alternative have a '!' modifier.
2363     *
2364     * If an alternative defeats the default tree construction, it
2365     * must set <rule>_AST to the root of the returned AST.
2366     *
2367     * Each alternative that does automatic tree construction, builds
2368     * up root and child list pointers in an ASTPair structure.
2369     *
2370     * A rule finishes by setting the returnAST variable from the
2371     * ASTPair.
2372     *
2373     * @param rule The name of the rule to generate
2374     * @param startSymbol true if the rule is a start symbol (i.e., not referenced elsewhere)
2375     */
2376    public void genRule(RuleSymbol s, boolean startSymbol, int ruleNum) {
2377        tabs = 1;
2378
2379        if (DEBUG_CODE_GENERATOR) System.out.println("genRule(" + s.getId() + ")");
2380        if (!s.isDefined()) {
2381            antlrTool.error("undefined rule: " + s.getId());
2382            return;
2383        }
2384
2385        // Generate rule return type, name, arguments
2386        RuleBlock rblk = s.getBlock();
2387
2388        currentRule = rblk;
2389        currentASTResult = s.getId();
2390
2391        // clear list of declared ast variables..
2392        declaredASTVariables.clear();
2393
2394        // Save the AST generation state, and set it to that of the rule
2395        boolean savegenAST = genAST;
2396        genAST = genAST && rblk.getAutoGen();
2397
2398        // boolean oldsaveTest = saveText;
2399        saveText = rblk.getAutoGen();
2400
2401        // print javadoc comment if any
2402        if (s.comment != null) {
2403            _println(s.comment);
2404        }
2405
2406        // Gen method access and final qualifier
2407        print(s.access + " final ");
2408
2409        // Gen method return type (note lexer return action set at rule creation)
2410        if (rblk.returnAction != null) {
2411            // Has specified return value
2412            _print(extractTypeOfAction(rblk.returnAction, rblk.getLine(), rblk.getColumn()) + " ");
2413        }
2414        else {
2415            // No specified return value
2416            _print("void ");
2417        }
2418
2419        // Gen method name
2420        _print(s.getId() + "(");
2421
2422        // Additional rule parameters common to all rules for this grammar
2423        _print(commonExtraParams);
2424        if (commonExtraParams.length() != 0 && rblk.argAction != null) {
2425            _print(",");
2426        }
2427
2428        // Gen arguments
2429        if (rblk.argAction != null) {
2430            // Has specified arguments
2431            _println("");
2432            tabs++;
2433            println(rblk.argAction);
2434            tabs--;
2435            print(")");
2436        }
2437        else {
2438            // No specified arguments
2439            _print(")");
2440        }
2441
2442        // Gen throws clause and open curly
2443        _print(" throws " + exceptionThrown);
2444        if (grammar instanceof ParserGrammar) {
2445            _print(", TokenStreamException");
2446        }
2447        else if (grammar instanceof LexerGrammar) {
2448            _print(", CharStreamException, TokenStreamException");
2449        }
2450        // Add user-defined exceptions unless lexer (for now)
2451        if (rblk.throwsSpec != null) {
2452            if (grammar instanceof LexerGrammar) {
2453                antlrTool.error("user-defined throws spec not allowed (yet) for lexer rule " + rblk.ruleName);
2454            }
2455            else {
2456                _print(", " + rblk.throwsSpec);
2457            }
2458        }
2459
2460        _println(" {");
2461        tabs++;
2462
2463        // Convert return action to variable declaration
2464        if (rblk.returnAction != null)
2465            println(rblk.returnAction + ";");
2466
2467        // print out definitions needed by rules for various grammar types
2468        println(commonLocalVars);
2469
2470        if (grammar.traceRules) {
2471            if (grammar instanceof TreeWalkerGrammar) {
2472                println("traceIn(\"" + s.getId() + "\",_t);");
2473            }
2474            else {
2475                println("traceIn(\"" + s.getId() + "\");");
2476            }
2477        }
2478
2479        if (grammar instanceof LexerGrammar) {
2480            // lexer rule default return value is the rule's token name
2481            // This is a horrible hack to support the built-in EOF lexer rule.
2482            if (s.getId().equals("mEOF"))
2483                println("_ttype = Token.EOF_TYPE;");
2484            else
2485                println("_ttype = " + s.getId().substring(1) + ";");
2486            println("int _saveIndex;");     // used for element! (so we can kill text matched for element)
2487            /*
2488             println("boolean old_saveConsumedInput=saveConsumedInput;");
2489             if ( !rblk.getAutoGen() ) {        // turn off "save input" if ! on rule
2490             println("saveConsumedInput=false;");
2491             }
2492             */
2493        }
2494
2495        // if debugging, write code to mark entry to the rule
2496        if (grammar.debuggingOutput)
2497            if (grammar instanceof ParserGrammar)
2498                println("fireEnterRule(" + ruleNum + ",0);");
2499            else if (grammar instanceof LexerGrammar)
2500                println("fireEnterRule(" + ruleNum + ",_ttype);");
2501
2502        // Generate trace code if desired
2503        if (grammar.debuggingOutput || grammar.traceRules) {
2504            println("try { // debugging");
2505            tabs++;
2506        }
2507
2508        // Initialize AST variables
2509        if (grammar instanceof TreeWalkerGrammar) {
2510            // "Input" value for rule
2511        println(labeledElementASTType + " " + s.getId() + "_AST_in = (_t == ASTNULL) ? null : (" + labeledElementASTType + ")_t;");
2512        }
2513        if (grammar.buildAST) {
2514            // Parser member used to pass AST returns from rule invocations
2515            println("returnAST = null;");
2516            // Tracks AST construction
2517            // println("ASTPair currentAST = (inputState.guessing==0) ? new ASTPair() : null;");
2518            println("ASTPair currentAST = new ASTPair();");
2519            // User-settable return value for rule.
2520            println(labeledElementASTType + " " + s.getId() + "_AST = null;");
2521        }
2522
2523        genBlockPreamble(rblk);
2524        genBlockInitAction(rblk);
2525        println("");
2526
2527        // Search for an unlabeled exception specification attached to the rule
2528        ExceptionSpec unlabeledUserSpec = rblk.findExceptionSpec("");
2529
2530        // Generate try block around the entire rule for  error handling
2531        if (unlabeledUserSpec != null || rblk.getDefaultErrorHandler()) {
2532            println("try {      // for error handling");
2533            tabs++;
2534        }
2535
2536        // Generate the alternatives
2537        if (rblk.alternatives.size() == 1) {
2538            // One alternative -- use simple form
2539            Alternative alt = rblk.getAlternativeAt(0);
2540            String   pred = alt.semPred;
2541            if (pred != null)
2542                genSemPred(pred, currentRule.line);
2543            if (alt.synPred != null) {
2544                antlrTool.warning(
2545                    "Syntactic predicate ignored for single alternative",
2546                    grammar.getFilename(),
2547                    alt.synPred.getLine(),
2548                    alt.synPred.getColumn()
2549                );
2550            }
2551            genAlt(alt, rblk);
2552        }
2553        else {
2554            // Multiple alternatives -- generate complex form
2555            boolean ok = grammar.theLLkAnalyzer.deterministic(rblk);
2556
2557            JavaBlockFinishingInfo howToFinish = genCommonBlock(rblk, false);
2558            genBlockFinish(howToFinish, throwNoViable);
2559        }
2560
2561        // Generate catch phrase for error handling
2562        if (unlabeledUserSpec != null || rblk.getDefaultErrorHandler()) {
2563            // Close the try block
2564            tabs--;
2565            println("}");
2566        }
2567
2568        // Generate user-defined or default catch phrases
2569        if (unlabeledUserSpec != null) {
2570            genErrorHandler(unlabeledUserSpec);
2571        }
2572        else if (rblk.getDefaultErrorHandler()) {
2573            // Generate default catch phrase
2574            println("catch (" + exceptionThrown + " ex) {");
2575            tabs++;
2576            // Generate code to handle error if not guessing
2577            if (grammar.hasSyntacticPredicate) {
2578                println("if (inputState.guessing==0) {");
2579                tabs++;
2580            }
2581            println("reportError(ex);");
2582            if (!(grammar instanceof TreeWalkerGrammar)) {
2583                // Generate code to consume until token in k==1 follow set
2584                Lookahead follow = grammar.theLLkAnalyzer.FOLLOW(1, rblk.endNode);
2585                String   followSetName = getBitsetName(markBitsetForGen(follow.fset));
2586                println("consume();");
2587                println("consumeUntil(" + followSetName + ");");
2588            }
2589            else {
2590                // Just consume one token
2591                println("if (_t!=null) {_t = _t.getNextSibling();}");
2592            }
2593            if (grammar.hasSyntacticPredicate) {
2594                tabs--;
2595                // When guessing, rethrow exception
2596                println("} else {");
2597                println("  throw ex;");
2598                println("}");
2599            }
2600            // Close catch phrase
2601            tabs--;
2602            println("}");
2603        }
2604
2605        // Squirrel away the AST "return" value
2606        if (grammar.buildAST) {
2607            println("returnAST = " + s.getId() + "_AST;");
2608        }
2609
2610        // Set return tree value for tree walkers
2611        if (grammar instanceof TreeWalkerGrammar) {
2612            println("_retTree = _t;");
2613        }
2614
2615        // Generate literals test for lexer rules so marked
2616        if (rblk.getTestLiterals()) {
2617            if (s.access.equals("protected")) {
2618                genLiteralsTestForPartialToken();
2619            }
2620            else {
2621                genLiteralsTest();
2622            }
2623        }
2624
2625        // if doing a lexer rule, dump code to create token if necessary
2626        if (grammar instanceof LexerGrammar) {
2627            println("if ( _createToken && _token==null && _ttype!=Token.SKIP ) {");
2628            println("   _token = makeToken(_ttype);");
2629            println("   _token.setText(new String(text.getBuffer(), _begin, text.length()-_begin));");
2630            println("}");
2631            println("_returnToken = _token;");
2632        }
2633
2634        // Gen the return statement if there is one (lexer has hard-wired return action)
2635        if (rblk.returnAction != null) {
2636            println("return " + extractIdOfAction(rblk.returnAction, rblk.getLine(), rblk.getColumn()) + ";");
2637        }
2638
2639        if (grammar.debuggingOutput || grammar.traceRules) {
2640            tabs--;
2641            println("} finally { // debugging");
2642            tabs++;
2643
2644            // If debugging, generate calls to mark exit of rule
2645            if (grammar.debuggingOutput)
2646                if (grammar instanceof ParserGrammar)
2647                    println("fireExitRule(" + ruleNum + ",0);");
2648                else if (grammar instanceof LexerGrammar)
2649                    println("fireExitRule(" + ruleNum + ",_ttype);");
2650
2651            if (grammar.traceRules) {
2652                if (grammar instanceof TreeWalkerGrammar) {
2653                    println("traceOut(\"" + s.getId() + "\",_t);");
2654                }
2655                else {
2656                    println("traceOut(\"" + s.getId() + "\");");
2657                }
2658            }
2659
2660            tabs--;
2661            println("}");
2662        }
2663
2664        tabs--;
2665        println("}");
2666        println("");
2667
2668        // Restore the AST generation state
2669        genAST = savegenAST;
2670
2671        // restore char save state
2672        // saveText = oldsaveTest;
2673    }
2674
2675    private void GenRuleInvocation(RuleRefElement rr) {
2676        // dump rule name
2677        _print(rr.targetRule + "(");
2678
2679        // lexers must tell rule if it should set _returnToken
2680        if (grammar instanceof LexerGrammar) {
2681            // if labeled, could access Token, so tell rule to create
2682            if (rr.getLabel() != null) {
2683                _print("true");
2684            }
2685            else {
2686                _print("false");
2687            }
2688            if (commonExtraArgs.length() != 0 || rr.args != null) {
2689                _print(",");
2690            }
2691        }
2692
2693        // Extra arguments common to all rules for this grammar
2694        _print(commonExtraArgs);
2695        if (commonExtraArgs.length() != 0 && rr.args != null) {
2696            _print(",");
2697        }
2698
2699        // Process arguments to method, if any
2700        RuleSymbol rs = (RuleSymbol)grammar.getSymbol(rr.targetRule);
2701        if (rr.args != null) {
2702            // When not guessing, execute user arg action
2703            ActionTransInfo tInfo = new ActionTransInfo();
2704            String   args = processActionForSpecialSymbols(rr.args, 0, currentRule, tInfo);
2705            if (tInfo.assignToRoot || tInfo.refRuleRoot != null) {
2706                antlrTool.error("Arguments of rule reference '" + rr.targetRule + "' cannot set or ref #" +
2707                           currentRule.getRuleName(), grammar.getFilename(), rr.getLine(), rr.getColumn());
2708            }
2709            _print(args);
2710
2711            // Warn if the rule accepts no arguments
2712            if (rs.block.argAction == null) {
2713                antlrTool.warning("Rule '" + rr.targetRule + "' accepts no arguments", grammar.getFilename(), rr.getLine(), rr.getColumn());
2714            }
2715        }
2716        else {
2717            // For C++, no warning if rule has parameters, because there may be default
2718            // values for all of the parameters
2719            if (rs.block.argAction != null) {
2720                antlrTool.warning("Missing parameters on reference to rule " + rr.targetRule, grammar.getFilename(), rr.getLine(), rr.getColumn());
2721            }
2722        }
2723        _println(");");
2724
2725        // move down to the first child while parsing
2726        if (grammar instanceof TreeWalkerGrammar) {
2727            println("_t = _retTree;");
2728        }
2729    }
2730
2731    protected void genSemPred(String   pred, int line) {
2732        // translate $ and # references
2733        ActionTransInfo tInfo = new ActionTransInfo();
2734        pred = processActionForSpecialSymbols(pred, line, currentRule, tInfo);
2735        // ignore translation info...we don't need to do anything with it.
2736        String   escapedPred = charFormatter.escapeString(pred);
2737
2738        // if debugging, wrap the semantic predicate evaluation in a method
2739        // that can tell SemanticPredicateListeners the result
2740        if (grammar.debuggingOutput && ((grammar instanceof ParserGrammar) || (grammar instanceof LexerGrammar)))
2741            pred = "fireSemanticPredicateEvaluated(persistence.antlr.debug.SemanticPredicateEvent.VALIDATING,"
2742                + addSemPred(escapedPred) + "," + pred + ")";
2743        println("if (!(" + pred + "))");
2744        println("  throw new SemanticException(\"" + escapedPred + "\");");
2745    }
2746
2747    /** Write an array of Strings which are the semantic predicate
2748     *  expressions.  The debugger will reference them by number only
2749     */
2750    protected void genSemPredMap() {
2751        Enumeration   e = semPreds.elements();
2752        println("private String _semPredNames[] = {");
2753        while (e.hasMoreElements())
2754            println("\"" + e.nextElement() + "\",");
2755        println("};");
2756    }
2757
2758    protected void genSynPred(SynPredBlock blk, String   lookaheadExpr) {
2759        if (DEBUG_CODE_GENERATOR) System.out.println("gen=>(" + blk + ")");
2760
2761        // Dump synpred result variable
2762        println("boolean synPredMatched" + blk.ID + " = false;");
2763        // Gen normal lookahead test
2764        println("if (" + lookaheadExpr + ") {");
2765        tabs++;
2766
2767        // Save input state
2768        if (grammar instanceof TreeWalkerGrammar) {
2769            println("AST __t" + blk.ID + " = _t;");
2770        }
2771        else {
2772            println("int _m" + blk.ID + " = mark();");
2773        }
2774
2775        // Once inside the try, assume synpred works unless exception caught
2776        println("synPredMatched" + blk.ID + " = true;");
2777        println("inputState.guessing++;");
2778
2779        // if debugging, tell listeners that a synpred has started
2780        if (grammar.debuggingOutput && ((grammar instanceof ParserGrammar) ||
2781            (grammar instanceof LexerGrammar))) {
2782            println("fireSyntacticPredicateStarted();");
2783        }
2784
2785        syntacticPredLevel++;
2786        println("try {");
2787        tabs++;
2788        gen((AlternativeBlock)blk);     // gen code to test predicate
2789        tabs--;
2790        //println("System.out.println(\"pred "+blk+" succeeded\");");
2791        println("}");
2792        println("catch (" + exceptionThrown + " pe) {");
2793        tabs++;
2794        println("synPredMatched" + blk.ID + " = false;");
2795        //println("System.out.println(\"pred "+blk+" failed\");");
2796        tabs--;
2797        println("}");
2798
2799        // Restore input state
2800        if (grammar instanceof TreeWalkerGrammar) {
2801            println("_t = __t" + blk.ID + ";");
2802        }
2803        else {
2804            println("rewind(_m" + blk.ID + ");");
2805        }
2806
2807        println("inputState.guessing--;");
2808
2809        // if debugging, tell listeners how the synpred turned out
2810        if (grammar.debuggingOutput && ((grammar instanceof ParserGrammar) ||
2811            (grammar instanceof LexerGrammar))) {
2812            println("if (synPredMatched" + blk.ID + ")");
2813            println("  fireSyntacticPredicateSucceeded();");
2814            println("else");
2815            println("  fireSyntacticPredicateFailed();");
2816        }
2817
2818        syntacticPredLevel--;
2819        tabs--;
2820
2821        // Close lookahead test
2822        println("}");
2823
2824        // Test synred result
2825        println("if ( synPredMatched" + blk.ID + " ) {");
2826    }
2827
2828    /** Generate a static array containing the names of the tokens,
2829     * indexed by the token type values.  This static array is used
2830     * to format error messages so that the token identifers or literal
2831     * strings are displayed instead of the token numbers.
2832     *
2833     * If a lexical rule has a paraphrase, use it rather than the
2834     * token label.
2835     */
2836    public void genTokenStrings() {
2837        // Generate a string for each token.  This creates a static
2838        // array of Strings indexed by token type.
2839        println("");
2840        println("public static final String[] _tokenNames = {");
2841        tabs++;
2842
2843        // Walk the token vocabulary and generate a Vector of strings
2844        // from the tokens.
2845        Vector v = grammar.tokenManager.getVocabulary();
2846        for (int i = 0; i < v.size(); i++) {
2847            String   s = (String  )v.elementAt(i);
2848            if (s == null) {
2849                s = "<" + String.valueOf(i) + ">";
2850            }
2851            if (!s.startsWith("\"") && !s.startsWith("<")) {
2852                TokenSymbol ts = (TokenSymbol)grammar.tokenManager.getTokenSymbol(s);
2853                if (ts != null && ts.getParaphrase() != null) {
2854                    s = StringUtils.stripFrontBack(ts.getParaphrase(), "\"", "\"");
2855                }
2856            }
2857            print(charFormatter.literalString(s));
2858            if (i != v.size() - 1) {
2859                _print(",");
2860            }
2861            _println("");
2862        }
2863
2864        // Close the string array initailizer
2865        tabs--;
2866        println("};");
2867    }
2868
2869    /** Create and set Integer token type objects that map
2870     *  to Java Class objects (which AST node to create).
2871     */
2872    protected void genTokenASTNodeMap() {
2873        println("");
2874        println("protected void buildTokenTypeASTClassMap() {");
2875        // Generate a map.put("T","TNode") for each token
2876        // if heterogeneous node known for that token T.
2877        tabs++;
2878        boolean generatedNewHashtable = false;
2879        int n = 0;
2880        // Walk the token vocabulary and generate puts.
2881        Vector v = grammar.tokenManager.getVocabulary();
2882        for (int i = 0; i < v.size(); i++) {
2883            String   s = (String  )v.elementAt(i);
2884            if (s != null) {
2885                TokenSymbol ts = grammar.tokenManager.getTokenSymbol(s);
2886                if (ts != null && ts.getASTNodeType() != null) {
2887                    n++;
2888                    if ( !generatedNewHashtable ) {
2889                        // only generate if we are going to add a mapping
2890                        println("tokenTypeToASTClassMap = new Hashtable();");
2891                        generatedNewHashtable = true;
2892                    }
2893                    println("tokenTypeToASTClassMap.put(new Integer("+ts.getTokenType()+"), "+
2894                            ts.getASTNodeType()+".class);");
2895                }
2896            }
2897        }
2898
2899        if ( n==0 ) {
2900            println("tokenTypeToASTClassMap=null;");
2901        }
2902        tabs--;
2903        println("};");
2904    }
2905
2906    /** Generate the token types Java file */
2907    protected void genTokenTypes(TokenManager tm) throws IOException   {
2908        // Open the token output Java file and set the currentOutput stream
2909        // SAS: file open was moved to a method so a subclass can override
2910        //      This was mainly for the VAJ interface
2911        setupOutput(tm.getName() + TokenTypesFileSuffix);
2912
2913        tabs = 0;
2914
2915        // Generate the header common to all Java files
2916        genHeader();
2917        // Do not use printAction because we assume tabs==0
2918        println(behavior.getHeaderAction(""));
2919
2920        // Encapsulate the definitions in an interface.  This can be done
2921        // because they are all constants.
2922        println("public interface " + tm.getName() + TokenTypesFileSuffix + " {");
2923        tabs++;
2924
2925        // Generate a definition for each token type
2926        Vector v = tm.getVocabulary();
2927
2928        // Do special tokens manually
2929        println("int EOF = " + Token.EOF_TYPE + ";");
2930        println("int NULL_TREE_LOOKAHEAD = " + Token.NULL_TREE_LOOKAHEAD + ";");
2931
2932        for (int i = Token.MIN_USER_TYPE; i < v.size(); i++) {
2933            String   s = (String  )v.elementAt(i);
2934            if (s != null) {
2935                if (s.startsWith("\"")) {
2936                    // a string literal
2937                    StringLiteralSymbol sl = (StringLiteralSymbol)tm.getTokenSymbol(s);
2938                    if (sl == null) {
2939                        antlrTool.panic("String literal " + s + " not in symbol table");
2940                    }
2941                    else if (sl.label != null) {
2942                        println("int " + sl.label + " = " + i + ";");
2943                    }
2944                    else {
2945                        String   mangledName = mangleLiteral(s);
2946                        if (mangledName != null) {
2947                            // We were able to create a meaningful mangled token name
2948                            println("int " + mangledName + " = " + i + ";");
2949                            // if no label specified, make the label equal to the mangled name
2950                            sl.label = mangledName;
2951                        }
2952                        else {
2953                            println("// " + s + " = " + i);
2954                        }
2955                    }
2956                }
2957                else if (!s.startsWith("<")) {
2958                    println("int " + s + " = " + i + ";");
2959                }
2960            }
2961        }
2962
2963        // Close the interface
2964        tabs--;
2965        println("}");
2966
2967        // Close the tokens output file
2968        currentOutput.close();
2969        currentOutput = null;
2970        exitIfError();
2971    }
2972
2973    /** Get a string for an expression to generate creation of an AST subtree.
2974     * @param v A Vector of String, where each element is an expression in the target language yielding an AST node.
2975     */
2976    public String   getASTCreateString(Vector v) {
2977        if (v.size() == 0) {
2978            return "";
2979        }
2980        StringBuffer   buf = new StringBuffer  ();
2981        buf.append("(" + labeledElementASTType +
2982                   ")astFactory.make( (new ASTArray(" + v.size() +
2983                   "))");
2984        for (int i = 0; i < v.size(); i++) {
2985            buf.append(".add(" + v.elementAt(i) + ")");
2986        }
2987        buf.append(")");
2988        return buf.toString();
2989    }
2990
2991    /** Get a string for an expression to generate creating of an AST node
2992     * @param atom The grammar node for which you are creating the node
2993     * @param str The arguments to the AST constructor
2994     */
2995    public String   getASTCreateString(GrammarAtom atom, String   astCtorArgs) {
2996        //System.out.println("getASTCreateString("+atom+","+astCtorArgs+")");
2997        if (atom != null && atom.getASTNodeType() != null) {
2998            // they specified a type either on the reference or in tokens{} section
2999            return "("+atom.getASTNodeType()+")"+
3000                    "astFactory.create("+astCtorArgs+",\""+atom.getASTNodeType()+"\")";
3001        }
3002        else {
3003            // must be an action or something since not referencing an atom
3004            return getASTCreateString(astCtorArgs);
3005        }
3006    }
3007
3008    /** Get a string for an expression to generate creating of an AST node.
3009     *  Parse the first (possibly only) argument looking for the token type.
3010     *  If the token type is a valid token symbol, ask for it's AST node type
3011     *  and add to the end if only 2 arguments.  The forms are #[T], #[T,"t"],
3012     *  and as of 2.7.2 #[T,"t",ASTclassname].
3013     *
3014     * @param str The arguments to the AST constructor
3015     */
3016    public String   getASTCreateString(String   astCtorArgs) {
3017        //System.out.println("AST CTOR: "+astCtorArgs);
3018        if ( astCtorArgs==null ) {
3019            astCtorArgs = "";
3020        }
3021        int nCommas = 0;
3022        for (int i=0; i<astCtorArgs.length(); i++) {
3023            if ( astCtorArgs.charAt(i)==',' ) {
3024                nCommas++;
3025            }
3026        }
3027        //System.out.println("num commas="+nCommas);
3028        if ( nCommas<2 ) { // if 1 or 2 args
3029            int firstComma = astCtorArgs.indexOf(',');
3030            int lastComma = astCtorArgs.lastIndexOf(',');
3031            String   tokenName = astCtorArgs;
3032            if ( nCommas>0 ) {
3033                tokenName = astCtorArgs.substring(0,firstComma);
3034            }
3035            //System.out.println("Checking for ast node type of "+tokenName);
3036            TokenSymbol ts = grammar.tokenManager.getTokenSymbol(tokenName);
3037            if ( ts!=null ) {
3038                String   astNodeType = ts.getASTNodeType();
3039                //System.out.println("node type of "+tokenName+" is "+astNodeType);
3040                String   emptyText = "";
3041                if ( nCommas==0 ) {
3042                    // need to add 2nd arg of blank text for token text
3043                    emptyText = ",\"\"";
3044                }
3045                if ( astNodeType!=null ) {
3046                    return "("+astNodeType+")"+
3047                            "astFactory.create("+astCtorArgs+emptyText+",\""+astNodeType+"\")";
3048                }
3049                // fall through and just do a regular create with cast on front
3050                // if necessary (it differs from default "AST").
3051            }
3052            if ( labeledElementASTType.equals("AST") ) {
3053                return "astFactory.create("+astCtorArgs+")";
3054            }
3055            return "("+labeledElementASTType+")"+
3056                    "astFactory.create("+astCtorArgs+")";
3057        }
3058        // create default type or (since 2.7.2) 3rd arg is classname
3059        return "(" + labeledElementASTType + ")astFactory.create(" + astCtorArgs + ")";
3060    }
3061
3062    protected String   getLookaheadTestExpression(Lookahead[] look, int k) {
3063        StringBuffer   e = new StringBuffer  (100);
3064        boolean first = true;
3065
3066        e.append("(");
3067        for (int i = 1; i <= k; i++) {
3068            BitSet p = look[i].fset;
3069            if (!first) {
3070                e.append(") && (");
3071            }
3072            first = false;
3073
3074            // Syn preds can yield <end-of-syn-pred> (epsilon) lookahead.
3075            // There is no way to predict what that token would be.  Just
3076            // allow anything instead.
3077            if (look[i].containsEpsilon()) {
3078                e.append("true");
3079            }
3080            else {
3081                e.append(getLookaheadTestTerm(i, p));
3082            }
3083        }
3084        e.append(")");
3085
3086        return e.toString();
3087    }
3088
3089    /**Generate a lookahead test expression for an alternate.  This
3090     * will be a series of tests joined by '&&' and enclosed by '()',
3091     * the number of such tests being determined by the depth of the lookahead.
3092     */
3093    protected String   getLookaheadTestExpression(Alternative alt, int maxDepth) {
3094        int depth = alt.lookaheadDepth;
3095        if (depth == GrammarAnalyzer.NONDETERMINISTIC) {
3096            // if the decision is nondeterministic, do the best we can: LL(k)
3097            // any predicates that are around will be generated later.
3098            depth = grammar.maxk;
3099        }
3100
3101        if (maxDepth == 0) {
3102            // empty lookahead can result from alt with sem pred
3103            // that can see end of token.  E.g., A : {pred}? ('a')? ;
3104            return "( true )";
3105        }
3106
3107        return "(" + getLookaheadTestExpression(alt.cache, depth) + ")";
3108    }
3109
3110    /**Generate a depth==1 lookahead test expression given the BitSet.
3111     * This may be one of:
3112     * 1) a series of 'x==X||' tests
3113     * 2) a range test using >= && <= where possible,
3114     * 3) a bitset membership test for complex comparisons
3115     * @param k The lookahead level
3116     * @param p The lookahead set for level k
3117     */
3118    protected String   getLookaheadTestTerm(int k, BitSet p) {
3119        // Determine the name of the item to be compared
3120        String   ts = lookaheadString(k);
3121
3122        // Generate a range expression if possible
3123        int[] elems = p.toArray();
3124        if (elementsAreRange(elems)) {
3125            return getRangeExpression(k, elems);
3126        }
3127
3128        // Generate a bitset membership test if possible
3129        StringBuffer   e;
3130        int degree = p.degree();
3131        if (degree == 0) {
3132            return "true";
3133        }
3134
3135        if (degree >= bitsetTestThreshold) {
3136            int bitsetIdx = markBitsetForGen(p);
3137            return getBitsetName(bitsetIdx) + ".member(" + ts + ")";
3138        }
3139
3140        // Otherwise, generate the long-winded series of "x==X||" tests
3141        e = new StringBuffer  ();
3142        for (int i = 0; i < elems.length; i++) {
3143            // Get the compared-to item (token or character value)
3144            String   cs = getValueString(elems[i]);
3145
3146            // Generate the element comparison
3147            if (i > 0) e.append("||");
3148            e.append(ts);
3149            e.append("==");
3150            e.append(cs);
3151        }
3152        return e.toString();
3153    }
3154
3155    /** Return an expression for testing a contiguous renage of elements
3156     * @param k The lookahead level
3157     * @param elems The elements representing the set, usually from BitSet.toArray().
3158     * @return String containing test expression.
3159     */
3160    public String   getRangeExpression(int k, int[] elems) {
3161        if (!elementsAreRange(elems)) {
3162            antlrTool.panic("getRangeExpression called with non-range");
3163        }
3164        int begin = elems[0];
3165        int end = elems[elems.length - 1];
3166        return
3167            "(" + lookaheadString(k) + " >= " + getValueString(begin) + " && " +
3168            lookaheadString(k) + " <= " + getValueString(end) + ")";
3169    }
3170
3171    /** getValueString: get a string representation of a token or char value
3172     * @param value The token or char value
3173     */
3174    private String   getValueString(int value) {
3175        String   cs;
3176        if (grammar instanceof LexerGrammar) {
3177            cs = charFormatter.literalChar(value);
3178        }
3179        else {
3180            TokenSymbol ts = grammar.tokenManager.getTokenSymbolAt(value);
3181            if (ts == null) {
3182                return "" + value; // return token type as string
3183                // tool.panic("vocabulary for token type " + value + " is null");
3184            }
3185            String   tId = ts.getId();
3186            if (ts instanceof StringLiteralSymbol) {
3187                // if string literal, use predefined label if any
3188                // if no predefined, try to mangle into LITERAL_xxx.
3189                // if can't mangle, use int value as last resort
3190                StringLiteralSymbol sl = (StringLiteralSymbol)ts;
3191                String   label = sl.getLabel();
3192                if (label != null) {
3193                    cs = label;
3194                }
3195                else {
3196                    cs = mangleLiteral(tId);
3197                    if (cs == null) {
3198                        cs = String.valueOf(value);
3199                    }
3200                }
3201            }
3202            else {
3203                cs = tId;
3204            }
3205        }
3206        return cs;
3207    }
3208
3209    /**Is the lookahead for this alt empty? */
3210    protected boolean lookaheadIsEmpty(Alternative alt, int maxDepth) {
3211        int depth = alt.lookaheadDepth;
3212        if (depth == GrammarAnalyzer.NONDETERMINISTIC) {
3213            depth = grammar.maxk;
3214        }
3215        for (int i = 1; i <= depth && i <= maxDepth; i++) {
3216            BitSet p = alt.cache[i].fset;
3217            if (p.degree() != 0) {
3218                return false;
3219            }
3220        }
3221        return true;
3222    }
3223
3224    private String   lookaheadString(int k) {
3225        if (grammar instanceof TreeWalkerGrammar) {
3226            return "_t.getType()";
3227        }
3228        return "LA(" + k + ")";
3229    }
3230
3231    /** Mangle a string literal into a meaningful token name.  This is
3232     * only possible for literals that are all characters.  The resulting
3233     * mangled literal name is literalsPrefix with the text of the literal
3234     * appended.
3235     * @return A string representing the mangled literal, or null if not possible.
3236     */
3237    private String   mangleLiteral(String   s) {
3238        String   mangled = antlrTool.literalsPrefix;
3239        for (int i = 1; i < s.length() - 1; i++) {
3240            if (!Character.isLetter(s.charAt(i)) &&
3241                s.charAt(i) != '_') {
3242                return null;
3243            }
3244            mangled += s.charAt(i);
3245        }
3246        if (antlrTool.upperCaseMangledLiterals) {
3247            mangled = mangled.toUpperCase();
3248        }
3249        return mangled;
3250    }
3251
3252    /** Map an identifier to it's corresponding tree-node variable.
3253     * This is context-sensitive, depending on the rule and alternative
3254     * being generated
3255     * @param idParam The identifier name to map
3256     * @return The mapped id (which may be the same as the input), or null if the mapping is invalid due to duplicates
3257     */
3258    public String   mapTreeId(String   idParam, ActionTransInfo transInfo) {
3259        // if not in an action of a rule, nothing to map.
3260        if (currentRule == null) return idParam;
3261
3262        boolean in_var = false;
3263        String   id = idParam;
3264        if (grammar instanceof TreeWalkerGrammar) {
3265            if (!grammar.buildAST) {
3266                in_var = true;
3267            }
3268            // If the id ends with "_in", then map it to the input variable
3269            else if (id.length() > 3 && id.lastIndexOf("_in") == id.length() - 3) {
3270                // Strip off the "_in"
3271                id = id.substring(0, id.length() - 3);
3272                in_var = true;
3273            }
3274        }
3275
3276        // Check the rule labels.  If id is a label, then the output
3277        // variable is label_AST, and the input variable is plain label.
3278        for (int i = 0; i < currentRule.labeledElements.size(); i++) {
3279            AlternativeElement elt = (AlternativeElement)currentRule.labeledElements.elementAt(i);
3280            if (elt.getLabel().equals(id)) {
3281                return in_var ? id : id + "_AST";
3282            }
3283        }
3284
3285        // Failing that, check the id-to-variable map for the alternative.
3286        // If the id is in the map, then output variable is the name in the
3287        // map, and input variable is name_in
3288        String   s = (String  )treeVariableMap.get(id);
3289        if (s != null) {
3290            if (s == NONUNIQUE) {
3291                // There is more than one element with this id
3292                antlrTool.error("Ambiguous reference to AST element "+id+
3293                                " in rule "+currentRule.getRuleName());
3294
3295                return null;
3296            }
3297            else if (s.equals(currentRule.getRuleName())) {
3298                // a recursive call to the enclosing rule is
3299                // ambiguous with the rule itself.
3300                antlrTool.error("Ambiguous reference to AST element "+id+
3301                                " in rule "+currentRule.getRuleName());
3302                return null;
3303            }
3304            else {
3305                return in_var ? s + "_in" : s;
3306            }
3307        }
3308
3309        // Failing that, check the rule name itself.  Output variable
3310        // is rule_AST; input variable is rule_AST_in (treeparsers).
3311        if (id.equals(currentRule.getRuleName())) {
3312            String   r = in_var ? id + "_AST_in" : id + "_AST";
3313            if (transInfo != null) {
3314                if (!in_var) {
3315                    transInfo.refRuleRoot = r;
3316                }
3317            }
3318            return r;
3319        }
3320        else {
3321            // id does not map to anything -- return itself.
3322            return id;
3323        }
3324    }
3325
3326    /** Given an element and the name of an associated AST variable,
3327     * create a mapping between the element "name" and the variable name.
3328     */
3329    private void mapTreeVariable(AlternativeElement e, String   name) {
3330        // For tree elements, defer to the root
3331        if (e instanceof TreeElement) {
3332            mapTreeVariable(((TreeElement)e).root, name);
3333            return;
3334        }
3335
3336        // Determine the name of the element, if any, for mapping purposes
3337        String   elName = null;
3338
3339        // Don't map labeled items
3340        if (e.getLabel() == null) {
3341            if (e instanceof TokenRefElement) {
3342                // use the token id
3343                elName = ((TokenRefElement)e).atomText;
3344            }
3345            else if (e instanceof RuleRefElement) {
3346                // use the rule name
3347                elName = ((RuleRefElement)e).targetRule;
3348            }
3349        }
3350        // Add the element to the tree variable map if it has a name
3351        if (elName != null) {
3352            if (treeVariableMap.get(elName) != null) {
3353                // Name is already in the map -- mark it as duplicate
3354                treeVariableMap.remove(elName);
3355                treeVariableMap.put(elName, NONUNIQUE);
3356            }
3357            else {
3358                treeVariableMap.put(elName, name);
3359            }
3360        }
3361    }
3362
3363    /** Lexically process $var and tree-specifiers in the action.
3364     *  This will replace #id and #(...) with the appropriate
3365     *  function calls and/or variables etc...
3366     */
3367    protected String   processActionForSpecialSymbols(String   actionStr,
3368                                                    int line,
3369                                                    RuleBlock currentRule,
3370                                                    ActionTransInfo tInfo) {
3371        if (actionStr == null || actionStr.length() == 0) return null;
3372
3373        // The action trans info tells us (at the moment) whether an
3374        // assignment was done to the rule's tree root.
3375        if (grammar == null)
3376            return actionStr;
3377
3378        // see if we have anything to do...
3379        if ((grammar.buildAST && actionStr.indexOf('#') != -1) ||
3380            grammar instanceof TreeWalkerGrammar ||
3381            ((grammar instanceof LexerGrammar ||
3382            grammar instanceof ParserGrammar)
3383            && actionStr.indexOf('$') != -1)) {
3384            // Create a lexer to read an action and return the translated version
3385            persistence.antlr.actions.java.ActionLexer lexer =
3386                    new persistence.antlr.actions.java.ActionLexer(actionStr,
3387                                                       currentRule,
3388                                                       this,
3389                                                       tInfo);
3390
3391            lexer.setLineOffset(line);
3392            lexer.setFilename(grammar.getFilename());
3393            lexer.setTool(antlrTool);
3394
3395            try {
3396                lexer.mACTION(true);
3397                actionStr = lexer.getTokenObject().getText();
3398                // System.out.println("action translated: "+actionStr);
3399                    // System.out.println("trans info is "+tInfo);
3400            }
3401            catch (RecognitionException ex) {
3402                lexer.reportError(ex);
3403                return actionStr;
3404            }
3405            catch (TokenStreamException tex) {
3406                antlrTool.panic("Error reading action:" + actionStr);
3407                return actionStr;
3408            }
3409            catch (CharStreamException io) {
3410                antlrTool.panic("Error reading action:" + actionStr);
3411                return actionStr;
3412            }
3413        }
3414        return actionStr;
3415    }
3416
3417    private void setupGrammarParameters(Grammar g) {
3418        if (g instanceof ParserGrammar) {
3419            labeledElementASTType = "AST";
3420            if (g.hasOption("ASTLabelType")) {
3421                Token tsuffix = g.getOption("ASTLabelType");
3422                if (tsuffix != null) {
3423                    String   suffix = StringUtils.stripFrontBack(tsuffix.getText(), "\"", "\"");
3424                    if (suffix != null) {
3425                        labeledElementASTType = suffix;
3426                    }
3427                }
3428            }
3429            labeledElementType = "Token ";
3430            labeledElementInit = "null";
3431            commonExtraArgs = "";
3432            commonExtraParams = "";
3433            commonLocalVars = "";
3434            lt1Value = "LT(1)";
3435            exceptionThrown = "RecognitionException";
3436            throwNoViable = "throw new NoViableAltException(LT(1), getFilename());";
3437        }
3438        else if (g instanceof LexerGrammar) {
3439            labeledElementType = "char ";
3440            labeledElementInit = "'\\0'";
3441            commonExtraArgs = "";
3442            commonExtraParams = "boolean _createToken";
3443            commonLocalVars = "int _ttype; Token _token=null; int _begin=text.length();";
3444            lt1Value = "LA(1)";
3445            exceptionThrown = "RecognitionException";
3446            throwNoViable = "throw new NoViableAltForCharException((char)LA(1), getFilename(), getLine(), getColumn());";
3447        }
3448        else if (g instanceof TreeWalkerGrammar) {
3449            labeledElementASTType = "AST";
3450            labeledElementType = "AST";
3451            if (g.hasOption("ASTLabelType")) {
3452                Token tsuffix = g.getOption("ASTLabelType");
3453                if (tsuffix != null) {
3454                    String   suffix = StringUtils.stripFrontBack(tsuffix.getText(), "\"", "\"");
3455                    if (suffix != null) {
3456                        labeledElementASTType = suffix;
3457                        labeledElementType = suffix;
3458                    }
3459                }
3460            }
3461            if (!g.hasOption("ASTLabelType")) {
3462                g.setOption("ASTLabelType", new Token(ANTLRTokenTypes.STRING_LITERAL, "AST"));
3463            }
3464            labeledElementInit = "null";
3465            commonExtraArgs = "_t";
3466            commonExtraParams = "AST _t";
3467            commonLocalVars = "";
3468            lt1Value = "(" + labeledElementASTType + ")_t";
3469            exceptionThrown = "RecognitionException";
3470            throwNoViable = "throw new NoViableAltException(_t);";
3471        }
3472        else {
3473            antlrTool.panic("Unknown grammar type");
3474        }
3475    }
3476
3477    /** This method exists so a subclass, namely VAJCodeGenerator,
3478     *  can open the file in its own evil way.  JavaCodeGenerator
3479     *  simply opens a text file...
3480     */
3481    public void setupOutput(String   className) throws IOException   {
3482        currentOutput = antlrTool.openOutputFile(className + ".java");
3483    }
3484}
3485
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags