Java > Open Source Codes > org > jacorb > idl > lexer


1 /*
2  * JacORB - a free Java ORB
3  *
4  * Copyright (C) 1997-2004 Gerald Brose.
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Library General Public
8  * License as published by the Free Software Foundation; either
9  * version 2 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Library General Public License for more details.
15  *
16  * You should have received a copy of the GNU Library General Public
17  * License along with this library; if not, write to the Free
18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19  */
20
21 package org.jacorb.idl;
22
23 import org.jacorb.idl.runtime.*;
24
25 import java.util.*;
26
27 /**
28  * This class implements a scanner (aka lexical analyzer or
29  * lexer) for IDL. The scanner reads characters from a global input
30  * stream and returns integers corresponding to the terminal number
31  * of the next token. Once the end of input is reached the EOF token
32  * is returned on every subsequent call.<p>
33  *
34  * All symbol constants are defined in sym.java which is generated by
35  * JavaCup from parser.cup.<p>
36  *
37  * In addition to the scanner proper (called first via init() then
38  * with next_token() to get each token) this class provides simple
39  * error and warning routines and keeps a count of errors and
40  * warnings that is publicly accessible. It also provides basic
41  * preprocessing facilties, i.e. it does handle preprocessor
42  * directives such as #define, #undef, #include, etc. although it
43  * does not provide full C++ preprocessing
44  *
45  * This class is "static" (i.e., it has only static members and methods).
46  *
47  * @version $Id: lexer.java,v 1.48 2004/05/06 12:39:59 nicolas Exp $
48  * @author Gerald Brose
49  *
50  */
51
52 public class lexer
53 {
54     private static org.apache.log.Logger logger = parser.getLogger();
55
56     /** First and second character of lookahead. */
57     protected static int next_char;
58     protected static int next_char2;
59
60     /** EOF constant. */
61     protected static final int EOF_CHAR = -1;
62
63     /**
64      * Table of keywords. Keywords are initially treated as
65      * identifiers. Just before they are returned we look them up in
66      * this table to see if they match one of the keywords. The
67      * string of the name is the key here, which indexes Integer
68      * objects holding the symbol number.
69      */
70
71     protected static Hashtable keywords = new Hashtable();
72
73     /** Table of keywords, stored in lower case. Keys are the
74      * lower case version of the keywords used as keys for the keywords
75      * hash above, and the values are the case sensitive versions of
76      * the keywords. This table is used for detecting collisions of
77      * identifiers with keywords.
78      */
79     protected static Hashtable keywords_lower_case = new Hashtable();
80
81
82     /** Table of Java reserved names.
83      */
84
85     protected static Hashtable java_keywords = new Hashtable();
86
87     /** Table of single character symbols. For ease of implementation, we
88      * store all unambiguous single character tokens in this table of Integer
89      * objects keyed by Integer objects with the numerical value of the
90      * appropriate char (currently Character objects have a bug which precludes
91      * their use in tables).
92      */
93     protected static Hashtable char_symbols = new Hashtable( 25 );
94
95
96     /** Defined symbols (preprocessor) */
97
98     protected static Hashtable defines = new Hashtable();
99     protected static boolean conditionalCompilation = true;
100
101     /** nested #ifdefs are pushed on this stack by the "preprocessor" */
102     private static java.util.Stack ifStack = new Stack();
103
104     private static java.util.Stack tokenStack = new Stack();
105
106     /** Current line number for use in error messages. */
107     protected static int current_line = 1;
108
109     /** Current line for use in error messages. */
110     protected static StringBuffer line = new StringBuffer ();
111
112     /** Character position in current line. */
113     protected static int current_position = 1;
114
115     /** Have we already read a '"' ? */
116     protected static boolean in_string = false;
117
118     /** Are we processing a wide char or string ? */
119     protected static boolean wide = false;
120
121     /** Count of total errors detected so far. */
122     static int error_count = 0;
123
124     /** Count of warnings issued so far */
125     public static int warning_count = 0;
126
127     /** currently active pragma prefix */
128     public static String currentPragmaPrefix = "";
129
130     /** current file name */
131     public static String currentFile = "";
132
133     /** reset the scanner state */
134
135     public static void reset()
136     {
137         current_position = 1;
138         error_count = 0;
139         warning_count = 0;
140         currentPragmaPrefix = "";
141         line = new StringBuffer ();
142         ifStack.removeAllElements();
143         tokenStack.removeAllElements();
144         defines.clear();
145     }
146
147     /**
148      * Initialize the scanner. This sets up the keywords and char_symbols
149      * tables and reads the first two characters of lookahead.
150      *
151      * "Object" is listed as reserved in the OMG spec.
152      * "int" is not, but I reserved it to bar its usage as a legal integer
153      * type.
154      */
155
156     public static void init()
157         throws java.io.IOException
158     {
159         /* set up standard symbols */
160         defines.put( "JACORB_IDL_1_4", "" );
161
162         /* set up the keyword table */
163
164         keywords.put( "abstract", new Integer ( sym.ABSTRACT ) );
165         keywords.put( "any", new Integer ( sym.ANY ) );
166         keywords.put( "attribute", new Integer ( sym.ATTRIBUTE ) );
167         keywords.put( "boolean", new Integer ( sym.BOOLEAN ) );
168         keywords.put( "case", new Integer ( sym.CASE ) );
169         keywords.put( "char", new Integer ( sym.CHAR ) );
170         keywords.put( "const", new Integer ( sym.CONST ) );
171         keywords.put( "context", new Integer ( sym.CONTEXT ) );
172         keywords.put( "custom", new Integer ( sym.CUSTOM ) );
173         keywords.put( "default", new Integer ( sym.DEFAULT ) );
174         keywords.put( "double", new Integer ( sym.DOUBLE ) );
175         keywords.put( "enum", new Integer ( sym.ENUM ) );
176         keywords.put( "exception", new Integer ( sym.EXCEPTION ) );
177         keywords.put( "factory", new Integer ( sym.FACTORY ) );
178         keywords.put( "FALSE", new Integer ( sym.FALSE ) );
179         keywords.put( "fixed", new Integer ( sym.FIXED ) );
180         keywords.put( "float", new Integer ( sym.FLOAT ) );
181         keywords.put( "in", new Integer ( sym.IN ) );
182         keywords.put( "inout", new Integer ( sym.INOUT ) );
183         keywords.put( "interface", new Integer ( sym.INTERFACE ) );
184         keywords.put( "local", new Integer ( sym.LOCAL ) );
185         keywords.put( "long", new Integer ( sym.LONG ) );
186         keywords.put( "module", new Integer ( sym.MODULE ) );
187         keywords.put( "native", new Integer ( sym.NATIVE ) );
188         keywords.put( "Object", new Integer ( sym.OBJECT ) );
189         keywords.put( "octet", new Integer ( sym.OCTET ) );
190         keywords.put( "oneway", new Integer ( sym.ONEWAY ) );
191         keywords.put( "out", new Integer ( sym.OUT ) );
192         keywords.put( "private", new Integer ( sym.PRIVATE ) );
193         keywords.put( "public", new Integer ( sym.PUBLIC ) );
194         keywords.put( "pseudo", new Integer ( sym.PSEUDO ) );
195         keywords.put( "raises", new Integer ( sym.RAISES ) );
196         keywords.put( "readonly", new Integer ( sym.READONLY ) );
197         keywords.put( "sequence", new Integer ( sym.SEQUENCE ) );
198         keywords.put( "short", new Integer ( sym.SHORT ) );
199         keywords.put( "string", new Integer ( sym.STRING ) );
200         keywords.put( "struct", new Integer ( sym.STRUCT ) );
201         keywords.put( "supports", new Integer ( sym.SUPPORTS ) );
202         keywords.put( "switch", new Integer ( sym.SWITCH ) );
203         keywords.put( "TRUE", new Integer ( sym.TRUE ) );
204         keywords.put( "truncatable", new Integer ( sym.TRUNCATABLE ) );
205         keywords.put( "typedef", new Integer ( sym.TYPEDEF ) );
206         keywords.put( "unsigned", new Integer ( sym.UNSIGNED ) );
207         keywords.put( "union", new Integer ( sym.UNION ) );
208         keywords.put( "ValueBase", new Integer ( sym.VALUEBASE ) );
209         keywords.put( "valuetype", new Integer ( sym.VALUETYPE ) );
210         keywords.put( "void", new Integer ( sym.VOID ) );
211         keywords.put( "wchar", new Integer ( sym.WCHAR ) );
212         keywords.put( "wstring", new Integer ( sym.WSTRING ) );
213
214         keywords.put( "::", new Integer ( sym.DBLCOLON ) );
215         keywords.put( "<<", new Integer ( sym.LSHIFT ) );
216         keywords.put( ">>", new Integer ( sym.RSHIFT ) );
217         keywords.put( "L\"", new Integer ( sym.LDBLQUOTE ) );
218
219         // setup the mapping of lower case keywords to case sensitive
220 // keywords
221
222         for( java.util.Enumeration e = keywords.keys(); e.hasMoreElements(); )
223         {
224             String keyword = (String )e.nextElement();
225             String keyword_lower_case = keyword.toLowerCase();
226             keywords_lower_case.put( keyword_lower_case, keyword );
227         }
228
229         /* set up the table of single character symbols */
230         char_symbols.put( new Integer ( ';' ), new Integer ( sym.SEMI ) );
231         char_symbols.put( new Integer ( ',' ), new Integer ( sym.COMMA ) );
232         char_symbols.put( new Integer ( '*' ), new Integer ( sym.STAR ) );
233         char_symbols.put( new Integer ( '.' ), new Integer ( sym.DOT ) );
234         char_symbols.put( new Integer ( ':' ), new Integer ( sym.COLON ) );
235         char_symbols.put( new Integer ( '=' ), new Integer ( sym.EQUALS ) );
236         char_symbols.put( new Integer ( '+' ), new Integer ( sym.PLUS ) );
237         char_symbols.put( new Integer ( '-' ), new Integer ( sym.MINUS ) );
238         char_symbols.put( new Integer ( '{' ), new Integer ( sym.LCBRACE ) );
239         char_symbols.put( new Integer ( '}' ), new Integer ( sym.RCBRACE ) );
240         char_symbols.put( new Integer ( '(' ), new Integer ( sym.LPAREN ) );
241         char_symbols.put( new Integer ( ')' ), new Integer ( sym.RPAREN ) );
242         char_symbols.put( new Integer ( '[' ), new Integer ( sym.LSBRACE ) );
243         char_symbols.put( new Integer ( ']' ), new Integer ( sym.RSBRACE ) );
244         char_symbols.put( new Integer ( '<' ), new Integer ( sym.LESSTHAN ) );
245         char_symbols.put( new Integer ( '>' ), new Integer ( sym.GREATERTHAN ) );
246         char_symbols.put( new Integer ( '\'' ), new Integer ( sym.QUOTE ) );
247         char_symbols.put( new Integer ( '\"' ), new Integer ( sym.DBLQUOTE ) );
248         char_symbols.put( new Integer ( '\\' ), new Integer ( sym.BSLASH ) );
249         char_symbols.put( new Integer ( '^' ), new Integer ( sym.CIRCUM ) );
250         char_symbols.put( new Integer ( '&' ), new Integer ( sym.AMPERSAND ) );
251         char_symbols.put( new Integer ( '/' ), new Integer ( sym.SLASH ) );
252         char_symbols.put( new Integer ( '%' ), new Integer ( sym.PERCENT ) );
253         char_symbols.put( new Integer ( '~' ), new Integer ( sym.TILDE ) );
254         char_symbols.put( new Integer ( '|' ), new Integer ( sym.BAR ) );
255         char_symbols.put( new Integer ( ' ' ), new Integer ( sym.SPACE ) );
256
257         /* set up reserved Java names */
258
259         java_keywords.put( "abstract", "" );
260         java_keywords.put( "boolean", "" );
261         java_keywords.put( "break", "" );
262         java_keywords.put( "byte", "" );
263         java_keywords.put( "case", "" );
264         java_keywords.put( "catch", "" );
265         java_keywords.put( "char", "" );
266         java_keywords.put( "class", "" );
267         java_keywords.put( "const", "" );
268         java_keywords.put( "continue", "" );
269         java_keywords.put( "default", "" );
270         java_keywords.put( "do", "" );
271         java_keywords.put( "double", "" );
272         java_keywords.put( "else", "" );
273         java_keywords.put( "extends", "" );
274         java_keywords.put( "false", "" );
275         java_keywords.put( "final", "" );
276         java_keywords.put( "finally", "" );
277         java_keywords.put( "float", "" );
278         java_keywords.put( "for", "" );
279         java_keywords.put( "goto", "" );
280         java_keywords.put( "if", "" );
281         java_keywords.put( "implements", "" );
282         java_keywords.put( "import", "" );
283         java_keywords.put( "instanceof", "" );
284         java_keywords.put( "int", "" );
285         java_keywords.put( "interface", "" );
286         java_keywords.put( "long", "" );
287         java_keywords.put( "native", "" );
288         java_keywords.put( "new", "" );
289         java_keywords.put( "null", "" );
290         java_keywords.put( "package", "" );
291         java_keywords.put( "private", "" );
292         java_keywords.put( "protected", "" );
293         java_keywords.put( "public", "" );
294         java_keywords.put( "return", "" );
295         java_keywords.put( "short", "" );
296         java_keywords.put( "static", "" );
297         java_keywords.put( "super", "" );
298         java_keywords.put( "switch", "" );
299         java_keywords.put( "synchronized", "" );
300         java_keywords.put( "true", "" );
301         java_keywords.put( "this", "" );
302         java_keywords.put( "throw", "" );
303         java_keywords.put( "throws", "" );
304         java_keywords.put( "transient", "" );
305         java_keywords.put( "try", "" );
306         java_keywords.put( "void", "" );
307         java_keywords.put( "volatile", "" );
308         java_keywords.put( "while", "" );
309
310         java_keywords.put( "clone", "" );
311         java_keywords.put( "equals", "" );
312         java_keywords.put( "finalize", "" );
313         java_keywords.put( "getClass", "" );
314         java_keywords.put( "hashCode", "" );
315         java_keywords.put( "notify", "" );
316         java_keywords.put( "notifyAll", "" );
317         java_keywords.put( "toString", "" );
318         java_keywords.put( "wait", "" );
319
320
321
322         /* stack needs a topmost value */
323         ifStack.push( new Boolean ( true ) );
324
325         /* read two characters of lookahead */
326
327         try
328         {
329             next_char = GlobalInputStream.read();
330         }
331         catch( Exception e )
332         {
333             org.jacorb.idl.parser.fatal_error( "Cannot read from file " +
334                     GlobalInputStream.currentFile().getAbsolutePath() +
335                     ", please check file name.", null );
336         }
337
338         if( next_char == EOF_CHAR )
339             next_char2 = EOF_CHAR;
340         else
341             next_char2 = GlobalInputStream.read();
342     }
343
344
345     public static void define( String symbol, String value )
346     {
347         if( logger.isDebugEnabled() )
348             logger.debug( "Defining: " + symbol + " as " + value );
349         defines.put( symbol, value );
350     }
351
352     public static void undefine( String symbol )
353     {
354         if( logger.isDebugEnabled() )
355             logger.debug( "Un-defining: " + symbol );
356         defines.remove( symbol );
357     }
358
359     public static String defined( String symbol )
360     {
361         return (String )defines.get( symbol );
362     }
363
364     /**
365      * record information about the last lexical scope so that it can be
366      * restored later
367      */
368
369     public static int currentLine()
370     {
371         return current_line;
372     }
373
374     /**
375      * return the current reading position
376      */
377
378     public static PositionInfo getPosition()
379     {
380         return new PositionInfo( current_line,
381                                  current_position,
382                                  currentPragmaPrefix,
383                                  line.toString(),
384                                  GlobalInputStream.currentFile() );
385     }
386
387     public static void restorePosition( PositionInfo p )
388     {
389         current_line = p.line_no;
390         currentPragmaPrefix = p.pragma_prefix;
391         current_position = 0;
392     }
393
394     /**
395      * Advance the scanner one character in the input stream. This moves
396      * next_char2 to next_char and then reads a new next_char2.
397      */
398
399     protected static void advance()
400         throws java.io.IOException
401     {
402         int old_char;
403
404         old_char = next_char;
405         next_char = next_char2;
406         next_char2 = GlobalInputStream.read();
407
408         line.append( (char)old_char );
409
410         /* count this */
411
412         current_position++;
413         if( old_char == '\n' )
414         {
415             current_line++;
416             current_position = 1;
417             line = new StringBuffer ();
418         }
419     }
420
421
422     /**
423      * Emit an error message. The message will be marked with both the
424      * current line number and the position in the line. Error messages
425      * are printed on standard error (System.err).
426      * @param message the message to print.
427      * @param p_info an optional PositionInfo object
428      */
429
430     public static void emit_error( String message )
431     {
432         if (parser.getLogger().isErrorEnabled())
433         {
434             logger.error( GlobalInputStream.currentFile().getAbsolutePath() +
435                            ", line: " + current_line +
436                            "(" + current_position + "): " +
437                            message + "\n\t" +
438                            line.toString() );
439         }
440         error_count++;
441     }
442
443     public static void emit_error( String message, str_token t )
444     {
445         if( t == null )
446         {
447             emit_error( message );
448         }
449         else
450         {
451             if (parser.getLogger().isErrorEnabled())
452             {
453                 logger.error( t.fileName + ", line:" + t.line_no +
454                               "(" + t.char_pos + "): " + message +
455                               "\n\t" + t.line_val );
456             }
457             error_count++;
458         }
459     }
460
461
462     /**
463      * Emit a warning message. The message will be marked with both the
464      * current line number and the position in the line. Messages are
465      * printed on standard error (System.err).
466      *
467      * @param message the message to print.
468      */
469
470     public static void emit_warn( String message )
471     {
472         if (parser.getLogger().isWarnEnabled())
473         {
474             logger.warn( message + " at " + current_line + "(" + current_position +
475                          "): \"" + line.toString() + "\"" );
476         }
477         warning_count++;
478     }
479
480
481     public static void emit_warn( String message, str_token t )
482     {
483         if( t == null )
484         {
485             emit_warn( message );
486         }
487         else
488         {
489             if (parser.getLogger().isWarnEnabled())
490             {
491                 logger.warn( " at " + t.fileName + ", line:" + t.line_no + "(" +
492                              t.char_pos + "): " + message + "\n\t" + t.line_val );
493             }
494
495             warning_count++;
496         }
497     }
498
499
500     /**
501      * Determine if a character is ok to start an id.
502      * @param ch the character in question.
503      */
504     protected static boolean id_start_char( int ch )
505     {
506         return
507                 ( ch >= 'a' && ch <= 'z' ) ||
508                 ( ch >= 'A' && ch <= 'Z' ) ||
509                 ( ch == '_' );
510     }
511
512
513     /**
514      * Determine if a character is ok for the middle of an id.
515      * @param ch the character in question.
516      */
517     protected static boolean id_char( int ch )
518     {
519         return id_start_char( ch ) || ( ch == '_' ) || ( ch >= '0' && ch <= '9' );
520     }
521
522
523     /**
524      * Try to look up a single character symbol, returns -1 for not found.
525      * @param ch the character in question.
526      */
527
528     protected static int find_single_char( int ch )
529     {
530         Integer result;
531
532         result = (Integer )char_symbols.get( new Integer ( (char)ch ) );
533         if( result == null )
534             return -1;
535         else
536             return result.intValue();
537     }
538
539     /**
540      * Handle swallowing up a comment. Both old style C and new style C++
541      * comments are handled.
542      */
543     protected static void swallow_comment()
544             throws java.io.IOException
545     {
546         /* next_char == '/' at this point */
547
548         /* is it a traditional comment */
549         if( next_char2 == '*' )
550         {
551             /* swallow the opener */
552             advance();
553             advance();
554
555             /* swallow the comment until end of comment or EOF */
556             for( ; ; )
557             {
558                 /* if its EOF we have an error */
559                 if( next_char == EOF_CHAR )
560                 {
561                     emit_error( "Specification file ends inside a comment", null );
562                     return;
563                 }
564
565                 /* if we can see the closer we are done */
566                 if( next_char == '*' && next_char2 == '/' )
567                 {
568                     advance();
569                     advance();
570                     return;
571                 }
572
573                 /* otherwise swallow char and move on */
574                 advance();
575             }
576         }
577
578         /* is its a new style comment */
579         if( next_char2 == '/' )
580         {
581             /* swallow the opener */
582             advance();
583             advance();
584
585             /* swallow to '\n', '\f', or EOF */
586             while( next_char != '\n' && next_char != '\f' && next_char != '\r' && next_char != EOF_CHAR )
587             {
588                 advance();
589             }
590
591             return;
592         }
593
594         /* shouldn't get here, but... if we get here we have an error */
595         emit_error( "Malformed comment in specification -- ignored", null );
596         advance();
597     }
598
599
600     /**
601      * Preprocessor directives are handled here.
602      */
603
604     protected static void preprocess()
605         throws java.io.IOException
606     {
607         for( ; ; )
608         {
609             /* if its EOF we have an error */
610             if( next_char == EOF_CHAR )
611             {
612                 emit_error( "Specification file ends inside a preprocessor directive", null );
613                 return;
614             }
615             else if( next_char != '#' )
616             {
617                 emit_error( "expected #, got " + (char)next_char + " instead!", null );
618             }
619             else
620                 advance(); // skip '#'
621
622             // the following is done to allow for # ifdef sloppiness
623 while( ( ' ' == next_char ) || ( '\t' == next_char ) )
624                 advance();
625
626             String dir = get_string();
627
628             if( dir.equals( "include" ) )
629             {
630                 if( !conditionalCompilation )
631                     return;
632                 advance(); // skip ' '
633 boolean useIncludePath = ( next_char == '<' );
634                 advance(); // skip `\"' or '<'
635
636                 String fname = get_string();
637
638                 if( useIncludePath && ( next_char != '>' ) )
639                     emit_error( "Syntax error in #include directive, expecting '>'" );
640                 else if( !useIncludePath && ( next_char != '\"' ) )
641                     emit_error( "Syntax error in #include directive, expecting \"" );
642
643                 /* swallow to '\n', '\f', or EOF */
644                 while( next_char != '\n' && next_char != '\f' && next_char != '\r' &&
645                         next_char != EOF_CHAR )
646                 {
647                     advance();
648                 }
649                 GlobalInputStream.include( fname, next_char2, useIncludePath );
650                 current_line = 0;
651                 advance();
652                 advance();
653                 return;
654             }
655             else if( dir.equals( "define" ) )
656             {
657                 if( !conditionalCompilation )
658                     return;
659                 swallow_whitespace();
660                 String name = get_string();
661                 StringBuffer text = new StringBuffer ();
662                 if( next_char == ' ' )
663                 {
664                     advance();
665                 }
666                 while( next_char != '\n' )
667                 {
668                     if( next_char == '\\' )
669                     {
670                         advance();
671                         advance();
672                     }
673                     text.append( (char)next_char );
674                     advance();
675                 }
676                 define( name, text.toString() );
677             }
678             else if( dir.equals( "error" ) )
679             {
680                 if( !conditionalCompilation )
681                     return;
682                 advance(); // skip ' '
683 String name = get_string();
684                 emit_error( name );
685             }
686             else if( dir.equals( "undef" ) )
687             {
688                 // Undefining symbol
689 if( !conditionalCompilation )
690                     return;
691                 swallow_whitespace();
692                 String name = get_string();
693                 undefine( name );
694             }
695             else if( dir.equals( "if" ) || dir.equals( "elif" ) )
696             {
697                 if (! dir.equals( "elif" ) )
698                 {
699                     ifStack.push( new Boolean ( conditionalCompilation ) );
700                     if( !conditionalCompilation )
701                         return;
702                 }
703
704                 swallow_whitespace();
705
706                 // the following snippet distinguishes between #if defined
707 // and #if !defined
708
709                 boolean straightDefined = true;
710                 if( '!' == next_char )
711                 {
712                     advance();
713                     straightDefined = false;
714                 }
715
716                 String defineStr = get_string_no_paren();
717
718                 if (defineStr.equals ("defined"))
719                 {
720                     swallow_whitespace();
721
722                     boolean brackets = ( '(' == next_char );
723                     if( brackets )
724                     {
725                         advance(); // skip '('
726 swallow_whitespace(); // any whitespace after ( ? skip it
727 }
728
729                     String name = get_string_no_paren();