CharScanner


1   package persistence.antlr;
2   
3   /* ANTLR Translator Generator
4    * Project led by Terence Parr at http://www.jGuru.com
5    * Software rights: http://www.antlr.org/license.html
6    *
7    */
8   
9   import java.util.Hashtable  ;
10  
11  import persistence.antlr.collections.impl.BitSet;
12  
13  import java.io.IOException  ;
14  
15  public abstract class CharScanner implements TokenStream {
16      static final char NO_CHAR = 0;
17      public static final char EOF_CHAR = (char)-1;
18      protected ANTLRStringBuffer text; // text of current token
19  
20      protected boolean saveConsumedInput = true; // does consume() save characters?
21      protected Class   tokenObjectClass; // what kind of tokens to create?
22      protected boolean caseSensitive = true;
23      protected boolean caseSensitiveLiterals = true;
24      protected Hashtable   literals; // set by subclass
25  
26      /** Tab chars are handled by tab() according to this value; override
27       *  method to do anything weird with tabs.
28       */
29      protected int tabsize = 8;
30  
31      protected Token _returnToken = null; // used to return tokens w/o using return val.
32  
33      // Hash string used so we don't new one every time to check literals table
34      protected ANTLRHashString hashString;
35  
36      protected LexerSharedInputState inputState;
37  
38      /** Used during filter mode to indicate that path is desired.
39       *  A subsequent scan error will report an error as usual if
40       *  acceptPath=true;
41       */
42      protected boolean commitToPath = false;
43  
44      /** Used to keep track of indentdepth for traceIn/Out */
45      protected int traceDepth = 0;
46  
47      public CharScanner() {
48          text = new ANTLRStringBuffer();
49          hashString = new ANTLRHashString(this);
50          setTokenObjectClass("persistence.antlr.CommonToken");
51      }
52  
53      public CharScanner(InputBuffer cb) { // SAS: use generic buffer
54          this();
55          inputState = new LexerSharedInputState(cb);
56      }
57  
58      public CharScanner(LexerSharedInputState sharedState) {
59          this();
60          inputState = sharedState;
61      }
62  
63      public void append(char c) {
64          if (saveConsumedInput) {
65              text.append(c);
66          }
67      }
68  
69      public void append(String   s) {
70          if (saveConsumedInput) {
71              text.append(s);
72          }
73      }
74  
75      public void commit() {
76          inputState.input.commit();
77      }
78  
79      public void consume() throws CharStreamException {
80          if (inputState.guessing == 0) {
81              char c = LA(1);
82              if (caseSensitive) {
83                  append(c);
84              }
85              else {
86                  // use input.LA(), not LA(), to get original case
87                  // CharScanner.LA() would toLower it.
88                  append(inputState.input.LA(1));
89              }
90              if (c == '\t') {
91                  tab();
92              }
93              else {
94                  inputState.column++;
95              }
96          }
97          inputState.input.consume();
98      }
99  
100     /** Consume chars until one matches the given char */
101     public void consumeUntil(int c) throws CharStreamException {
102         while (LA(1) != EOF_CHAR && LA(1) != c) {
103             consume();
104         }
105     }
106 
107     /** Consume chars until one matches the given set */
108     public void consumeUntil(BitSet set) throws CharStreamException {
109         while (LA(1) != EOF_CHAR && !set.member(LA(1))) {
110             consume();
111         }
112     }
113 
114     public boolean getCaseSensitive() {
115         return caseSensitive;
116     }
117 
118     public final boolean getCaseSensitiveLiterals() {
119         return caseSensitiveLiterals;
120     }
121 
122     public int getColumn() {
123         return inputState.column;
124     }
125 
126     public void setColumn(int c) {
127         inputState.column = c;
128     }
129 
130     public boolean getCommitToPath() {
131         return commitToPath;
132     }
133 
134     public String   getFilename() {
135         return inputState.filename;
136     }
137 
138     public InputBuffer getInputBuffer() {
139         return inputState.input;
140     }
141 
142     public LexerSharedInputState getInputState() {
143         return inputState;
144     }
145 
146     public void setInputState(LexerSharedInputState state) {
147         inputState = state;
148     }
149 
150     public int getLine() {
151         return inputState.line;
152     }
153 
154     /** return a copy of the current text buffer */
155     public String   getText() {
156         return text.toString();
157     }
158 
159     public Token getTokenObject() {
160         return _returnToken;
161     }
162 
163     public char LA(int i) throws CharStreamException {
164         if (caseSensitive) {
165             return inputState.input.LA(i);
166         }
167         else {
168             return toLower(inputState.input.LA(i));
169         }
170     }
171 
172     protected Token makeToken(int t) {
173         try {
174             Token tok = (Token)tokenObjectClass.newInstance();
175             tok.setType(t);
176             tok.setColumn(inputState.tokenStartColumn);
177             tok.setLine(inputState.tokenStartLine);
178             // tracking real start line now: tok.setLine(inputState.line);
179             return tok;
180         }
181         catch (InstantiationException   ie) {
182             panic("can't instantiate token: " + tokenObjectClass);
183         }
184         catch (IllegalAccessException   iae) {
185             panic("Token class is not accessible" + tokenObjectClass);
186         }
187         return Token.badToken;
188     }
189 
190     public int mark() {
191         return inputState.input.mark();
192     }
193 
194     public void match(char c) throws MismatchedCharException, CharStreamException {
195         if (LA(1) != c) {
196             throw new MismatchedCharException(LA(1), c, false, this);
197         }
198         consume();
199     }
200 
201     public void match(BitSet b) throws MismatchedCharException, CharStreamException {
202         if (!b.member(LA(1))) {
203             throw new MismatchedCharException(LA(1), b, false, this);
204         }
205         else {
206             consume();
207         }
208     }
209 
210     public void match(String   s) throws MismatchedCharException, CharStreamException {
211         int len = s.length();
212         for (int i = 0; i < len; i++) {
213             if (LA(1) != s.charAt(i)) {
214                 throw new MismatchedCharException(LA(1), s.charAt(i), false, this);
215             }
216             consume();
217         }
218     }
219 
220     public void matchNot(char c) throws MismatchedCharException, CharStreamException {
221         if (LA(1) == c) {
222             throw new MismatchedCharException(LA(1), c, true, this);
223         }
224         consume();
225     }
226 
227     public void matchRange(char c1, char c2) throws MismatchedCharException, CharStreamException {
228         if (LA(1) < c1 || LA(1) > c2) throw new MismatchedCharException(LA(1), c1, c2, false, this);
229         consume();
230     }
231 
232     public void newline() {
233         inputState.line++;
234         inputState.column = 1;
235     }
236 
237     /** advance the current column number by an appropriate amount
238      *  according to tab size. This method is called from consume().
239      */
240     public void tab() {
241         int c = getColumn();
242         int nc = ( ((c-1)/tabsize) + 1) * tabsize + 1;  // calculate tab stop
243         setColumn( nc );
244     }
245 
246     public void setTabSize( int size ) {
247         tabsize = size;
248     }
249 
250     public int getTabSize() {
251         return tabsize;
252     }
253 
254     /** @see #panic(String)
255      */
256     public void panic() {
257         System.err.println("CharScanner: panic");
258         System.exit(1);
259     }
260 
261     /** This method is executed by ANTLR internally when it detected an illegal
262      *  state that cannot be recovered from.
263      *  The default implementation of this method calls
264      *  {@link java.lang.System.exit(int)} and writes directly to
265      *  {@link java.lang.System.err)} , which is usually not appropriate when
266      *  a translator is embedded into a larger application. <em>It is highly
267      *  recommended that this method be overridden to handle the error in a
268      *  way appropriate for your application (e.g. throw an unchecked
269      *  exception)</em>.
270      */
271     public void panic(String   s) {
272         System.err.println("CharScanner; panic: " + s);
273         System.exit(1);
274     }
275 
276     /** Parser error-reporting function can be overridden in subclass */
277     public void reportError(RecognitionException ex) {
278         System.err.println(ex);
279     }
280 
281     /** Parser error-reporting function can be overridden in subclass */
282     public void reportError(String   s) {
283         if (getFilename() == null) {
284             System.err.println("error: " + s);
285         }
286         else {
287             System.err.println(getFilename() + ": error: " + s);
288         }
289     }
290 
291     /** Parser warning-reporting function can be overridden in subclass */
292     public void reportWarning(String   s) {
293         if (getFilename() == null) {
294             System.err.println("warning: " + s);
295         }
296         else {
297             System.err.println(getFilename() + ": warning: " + s);
298         }
299     }
300 
301     public void resetText() {
302         text.setLength(0);
303         inputState.tokenStartColumn = inputState.column;
304         inputState.tokenStartLine = inputState.line;
305     }
306 
307     public void rewind(int pos) {
308          inputState.input.rewind(pos);
309          // RK: should not be here, it is messing up column calculation
310          // setColumn(inputState.tokenStartColumn);
311     }
312 
313     public void setCaseSensitive(boolean t) {
314         caseSensitive = t;
315     }
316 
317     public void setCommitToPath(boolean commit) {
318         commitToPath = commit;
319     }
320 
321     public void setFilename(String   f) {
322         inputState.filename = f;
323     }
324 
325     public void setLine(int line) {
326         inputState.line = line;
327     }
328 
329     public void setText(String   s) {
330         resetText();
331         text.append(s);
332     }
333 
334     public void setTokenObjectClass(String   cl) {
335         try {
336             tokenObjectClass = Class.forName(cl);
337         }
338         catch (ClassNotFoundException   ce) {
339             panic("ClassNotFoundException: " + cl);
340         }
341     }
342 
343     // Test the token text against the literals table
344     // Override this method to perform a different literals test
345     public int testLiteralsTable(int ttype) {
346         hashString.setBuffer(text.getBuffer(), text.length());
347         Integer   literalsIndex = (Integer  )literals.get(hashString);
348         if (literalsIndex != null) {
349             ttype = literalsIndex.intValue();
350         }
351         return ttype;
352     }
353 
354     /** Test the text passed in against the literals table
355      * Override this method to perform a different literals test
356      * This is used primarily when you want to test a portion of
357      * a token.
358      */
359     public int testLiteralsTable(String   text, int ttype) {
360         ANTLRHashString s = new ANTLRHashString(text, this);
361         Integer   literalsIndex = (Integer  )literals.get(s);
362         if (literalsIndex != null) {
363             ttype = literalsIndex.intValue();
364         }
365         return ttype;
366     }
367 
368     // Override this method to get more specific case handling
369     public char toLower(char c) {
370         return Character.toLowerCase(c);
371     }
372 
373     public void traceIndent() {
374         for (int i = 0; i < traceDepth; i++)
375             System.out.print(" ");
376     }
377 
378     public void traceIn(String   rname) throws CharStreamException {
379         traceDepth += 1;
380         traceIndent();
381         System.out.println("> lexer " + rname + "; c==" + LA(1));
382     }
383 
384     public void traceOut(String   rname) throws CharStreamException {
385         traceIndent();
386         System.out.println("< lexer " + rname + "; c==" + LA(1));
387         traceDepth -= 1;
388     }
389 
390     /** This method is called by YourLexer.nextToken() when the lexer has
391      *  hit EOF condition.  EOF is NOT a character.
392      *  This method is not called if EOF is reached during
393      *  syntactic predicate evaluation or during evaluation
394      *  of normal lexical rules, which presumably would be
395      *  an IOException.  This traps the "normal" EOF condition.
396      *
397      *  uponEOF() is called after the complete evaluation of
398      *  the previous token and only if your parser asks
399      *  for another token beyond that last non-EOF token.
400      *
401      *  You might want to throw token or char stream exceptions
402      *  like: "Heh, premature eof" or a retry stream exception
403      *  ("I found the end of this file, go back to referencing file").
404      */
405     public void uponEOF() throws TokenStreamException, CharStreamException {
406     }
407 }
408
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags