StreamTokenizer


1   /*
2    * @(#)StreamTokenizer.java 1.44 04/05/18
3    *
4    * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
5    * SUN PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
6    */
7   
8   package java.io;
9   
10  
11  /**
12   * The <code>StreamTokenizer</code> class takes an input stream and
13   * parses it into "tokens", allowing the tokens to be
14   * read one at a time. The parsing process is controlled by a table
15   * and a number of flags that can be set to various states. The
16   * stream tokenizer can recognize identifiers, numbers, quoted
17   * strings, and various comment styles.
18   * <p>
19   * Each byte read from the input stream is regarded as a character
20   * in the range <code>'&#92;u0000'</code> through <code>'&#92;u00FF'</code>.
21   * The character value is used to look up five possible attributes of
22   * the character: <i>white space</i>, <i>alphabetic</i>,
23   * <i>numeric</i>, <i>string quote</i>, and <i>comment character</i>.
24   * Each character can have zero or more of these attributes.
25   * <p>
26   * In addition, an instance has four flags. These flags indicate:
27   * <ul>
28   * <li>Whether line terminators are to be returned as tokens or treated
29   *     as white space that merely separates tokens.
30   * <li>Whether C-style comments are to be recognized and skipped.
31   * <li>Whether C++-style comments are to be recognized and skipped.
32   * <li>Whether the characters of identifiers are converted to lowercase.
33   * </ul>
34   * <p>
35   * A typical application first constructs an instance of this class,
36   * sets up the syntax tables, and then repeatedly loops calling the
37   * <code>nextToken</code> method in each iteration of the loop until
38   * it returns the value <code>TT_EOF</code>.
39   *
40   * @author  James Gosling
41   * @version 1.44, 05/18/04
42   * @see     java.io.StreamTokenizer#nextToken()
43   * @see     java.io.StreamTokenizer#TT_EOF
44   * @since   JDK1.0
45   */
46  
47  public class StreamTokenizer {
48  
49      /* Only one of these will be non-null */
50      private Reader   reader = null;
51      private InputStream   input = null;
52  
53      private char buf[] = new char[20];
54  
55      /**
56       * The next character to be considered by the nextToken method.  May also
57       * be NEED_CHAR to indicate that a new character should be read, or SKIP_LF
58       * to indicate that a new character should be read and, if it is a '\n'
59       * character, it should be discarded and a second new character should be
60       * read.
61       */
62      private int peekc = NEED_CHAR;
63  
64      private static final int NEED_CHAR = Integer.MAX_VALUE;
65      private static final int SKIP_LF = Integer.MAX_VALUE - 1;
66  
67      private boolean pushedBack;
68      private boolean forceLower;
69      /** The line number of the last token read */
70      private int LINENO = 1;
71  
72      private boolean eolIsSignificantP = false;
73      private boolean slashSlashCommentsP = false;
74      private boolean slashStarCommentsP = false;
75  
76      private byte ctype[] = new byte[256];
77      private static final byte CT_WHITESPACE = 1;
78      private static final byte CT_DIGIT = 2;
79      private static final byte CT_ALPHA = 4;
80      private static final byte CT_QUOTE = 8;
81      private static final byte CT_COMMENT = 16;
82  
83      /**
84       * After a call to the <code>nextToken</code> method, this field
85       * contains the type of the token just read. For a single character
86       * token, its value is the single character, converted to an integer.
87       * For a quoted string token (see , its value is the quote character.
88       * Otherwise, its value is one of the following:
89       * <ul>
90       * <li><code>TT_WORD</code> indicates that the token is a word.
91       * <li><code>TT_NUMBER</code> indicates that the token is a number.
92       * <li><code>TT_EOL</code> indicates that the end of line has been read.
93       *     The field can only have this value if the
94       *     <code>eolIsSignificant</code> method has been called with the
95       *     argument <code>true</code>.
96       * <li><code>TT_EOF</code> indicates that the end of the input stream
97       *     has been reached.
98       * </ul>
99       * <p>
100      * The initial value of this field is -4.
101      *
102      * @see     java.io.StreamTokenizer#eolIsSignificant(boolean)
103      * @see     java.io.StreamTokenizer#nextToken()
104      * @see     java.io.StreamTokenizer#quoteChar(int)
105      * @see     java.io.StreamTokenizer#TT_EOF
106      * @see     java.io.StreamTokenizer#TT_EOL
107      * @see     java.io.StreamTokenizer#TT_NUMBER
108      * @see     java.io.StreamTokenizer#TT_WORD
109      */
110     public int ttype = TT_NOTHING;
111 
112     /**
113      * A constant indicating that the end of the stream has been read.
114      */
115     public static final int TT_EOF = -1;
116 
117     /**
118      * A constant indicating that the end of the line has been read.
119      */
120     public static final int TT_EOL = '\n';
121 
122     /**
123      * A constant indicating that a number token has been read.
124      */
125     public static final int TT_NUMBER = -2;
126 
127     /**
128      * A constant indicating that a word token has been read.
129      */
130     public static final int TT_WORD = -3;
131 
132     /* A constant indicating that no token has been read, used for
133      * initializing ttype.  FIXME This could be made public and
134      * made available as the part of the API in a future release.
135      */
136     private static final int TT_NOTHING = -4;
137 
138     /**
139      * If the current token is a word token, this field contains a
140      * string giving the characters of the word token. When the current
141      * token is a quoted string token, this field contains the body of
142      * the string.
143      * <p>
144      * The current token is a word when the value of the
145      * <code>ttype</code> field is <code>TT_WORD</code>. The current token is
146      * a quoted string token when the value of the <code>ttype</code> field is
147      * a quote character.
148      * <p>
149      * The initial value of this field is null.
150      *
151      * @see     java.io.StreamTokenizer#quoteChar(int)
152      * @see     java.io.StreamTokenizer#TT_WORD
153      * @see     java.io.StreamTokenizer#ttype
154      */
155     public String   sval;
156 
157     /**
158      * If the current token is a number, this field contains the value
159      * of that number. The current token is a number when the value of
160      * the <code>ttype</code> field is <code>TT_NUMBER</code>.
161      * <p>
162      * The initial value of this field is 0.0.
163      *
164      * @see     java.io.StreamTokenizer#TT_NUMBER
165      * @see     java.io.StreamTokenizer#ttype
166      */
167     public double nval;
168 
169     /** Private constructor that initializes everything except the streams. */
170     private StreamTokenizer() {
171     wordChars('a', 'z');
172     wordChars('A', 'Z');
173     wordChars(128 + 32, 255);
174     whitespaceChars(0, ' ');
175     commentChar('/');
176     quoteChar('"');
177     quoteChar('\'');
178     parseNumbers();
179     }
180 
181     /**
182      * Creates a stream tokenizer that parses the specified input
183      * stream. The stream tokenizer is initialized to the following
184      * default state:
185      * <ul>
186      * <li>All byte values <code>'A'</code> through <code>'Z'</code>,
187      *     <code>'a'</code> through <code>'z'</code>, and
188      *     <code>'&#92;u00A0'</code> through <code>'&#92;u00FF'</code> are
189      *     considered to be alphabetic.
190      * <li>All byte values <code>'&#92;u0000'</code> through
191      *     <code>'&#92;u0020'</code> are considered to be white space.
192      * <li><code>'/'</code> is a comment character.
193      * <li>Single quote <code>'&#92;''</code> and double quote <code>'"'</code>
194      *     are string quote characters.
195      * <li>Numbers are parsed.
196      * <li>Ends of lines are treated as white space, not as separate tokens.
197      * <li>C-style and C++-style comments are not recognized.
198      * </ul>
199      *
200      * @deprecated As of JDK version 1.1, the preferred way to tokenize an
201      * input stream is to convert it into a character stream, for example:
202      * <blockquote><pre>
203      *   Reader r = new BufferedReader(new InputStreamReader(is));
204      *   StreamTokenizer st = new StreamTokenizer(r);
205      * </pre></blockquote>
206      *
207      * @param      is        an input stream.
208      * @see        java.io.BufferedReader
209      * @see        java.io.InputStreamReader
210      * @see        java.io.StreamTokenizer#StreamTokenizer(java.io.Reader)
211      */
212     @Deprecated  
213     public StreamTokenizer(InputStream   is) {
214     this();
215         if (is == null) {
216             throw new NullPointerException  ();
217         }
218     input = is;
219     }
220 
221     /**
222      * Create a tokenizer that parses the given character stream.
223      *
224      * @param r  a Reader object providing the input stream.
225      * @since   JDK1.1
226      */
227     public StreamTokenizer(Reader   r) {
228     this();
229         if (r == null) {
230             throw new NullPointerException  ();
231         }
232     reader = r;
233     }
234 
235     /**
236      * Resets this tokenizer's syntax table so that all characters are
237      * "ordinary." See the <code>ordinaryChar</code> method
238      * for more information on a character being ordinary.
239      *
240      * @see     java.io.StreamTokenizer#ordinaryChar(int)
241      */
242     public void resetSyntax() {
243     for (int i = ctype.length; --i >= 0;)
244         ctype[i] = 0;
245     }
246 
247     /**
248      * Specifies that all characters <i>c</i> in the range
249      * <code>low&nbsp;&lt;=&nbsp;<i>c</i>&nbsp;&lt;=&nbsp;high</code>
250      * are word constituents. A word token consists of a word constituent
251      * followed by zero or more word constituents or number constituents.
252      *
253      * @param   low   the low end of the range.
254      * @param   hi    the high end of the range.
255      */
256     public void wordChars(int low, int hi) {
257     if (low < 0)
258         low = 0;
259     if (hi >= ctype.length)
260         hi = ctype.length - 1;
261     while (low <= hi)
262         ctype[low++] |= CT_ALPHA;
263     }
264 
265     /**
266      * Specifies that all characters <i>c</i> in the range
267      * <code>low&nbsp;&lt;=&nbsp;<i>c</i>&nbsp;&lt;=&nbsp;high</code>
268      * are white space characters. White space characters serve only to
269      * separate tokens in the input stream.
270      *
271      * <p>Any other attribute settings for the characters in the specified
272      * range are cleared.
273      *
274      * @param   low   the low end of the range.
275      * @param   hi    the high end of the range.
276      */
277     public void whitespaceChars(int low, int hi) {
278     if (low < 0)
279         low = 0;
280     if (hi >= ctype.length)
281         hi = ctype.length - 1;
282     while (low <= hi)
283         ctype[low++] = CT_WHITESPACE;
284     }
285 
286     /**
287      * Specifies that all characters <i>c</i> in the range
288      * <code>low&nbsp;&lt;=&nbsp;<i>c</i>&nbsp;&lt;=&nbsp;high</code>
289      * are "ordinary" in this tokenizer. See the
290      * <code>ordinaryChar</code> method for more information on a
291      * character being ordinary.
292      *
293      * @param   low   the low end of the range.
294      * @param   hi    the high end of the range.
295      * @see     java.io.StreamTokenizer#ordinaryChar(int)
296      */
297     public void ordinaryChars(int low, int hi) {
298     if (low < 0)
299         low = 0;
300     if (hi >= ctype.length)
301         hi = ctype.length - 1;
302     while (low <= hi)
303         ctype[low++] = 0;
304     }
305 
306     /**
307      * Specifies that the character argument is "ordinary"
308      * in this tokenizer. It removes any special significance the
309      * character has as a comment character, word component, string
310      * delimiter, white space, or number character. When such a character
311      * is encountered by the parser, the parser treats it as a
312      * single-character token and sets <code>ttype</code> field to the
313      * character value.
314      *
315      * <p>Making a line terminator character "ordinary" may interfere
316      * with the ability of a <code>StreamTokenizer</code> to count
317      * lines. The <code>lineno</code> method may no longer reflect
318      * the presence of such terminator characters in its line count.
319      *
320      * @param   ch   the character.
321      * @see     java.io.StreamTokenizer#ttype
322      */
323     public void ordinaryChar(int ch) {
324         if (ch >= 0 && ch < ctype.length)
325         ctype[ch] = 0;
326     }
327 
328     /**
329      * Specified that the character argument starts a single-line
330      * comment. All characters from the comment character to the end of
331      * the line are ignored by this stream tokenizer.
332      *
333      * <p>Any other attribute settings for the specified character are cleared.
334      *
335      * @param   ch   the character.
336      */
337     public void commentChar(int ch) {
338         if (ch >= 0 && ch < ctype.length)
339         ctype[ch] = CT_COMMENT;
340     }
341 
342     /**
343      * Specifies that matching pairs of this character delimit string
344      * constants in this tokenizer.
345      * <p>
346      * When the <code>nextToken</code> method encounters a string
347      * constant, the <code>ttype</code> field is set to the string
348      * delimiter and the <code>sval</code> field is set to the body of
349      * the string.
350      * <p>
351      * If a string quote character is encountered, then a string is
352      * recognized, consisting of all characters after (but not including)
353      * the string quote character, up to (but not including) the next
354      * occurrence of that same string quote character, or a line
355      * terminator, or end of file. The usual escape sequences such as
356      * <code>"&#92;n"</code> and <code>"&#92;t"</code> are recognized and
357      * converted to single characters as the string is parsed.
358      *
359      * <p>Any other attribute settings for the specified character are cleared.
360      *
361      * @param   ch   the character.
362      * @see     java.io.StreamTokenizer#nextToken()
363      * @see     java.io.StreamTokenizer#sval
364      * @see     java.io.StreamTokenizer#ttype
365      */
366     public void quoteChar(int ch) {
367         if (ch >= 0 && ch < ctype.length)
368         ctype[ch] = CT_QUOTE;
369     }
370 
371     /**
372      * Specifies that numbers should be parsed by this tokenizer. The
373      * syntax table of this tokenizer is modified so that each of the twelve
374      * characters:
375      * <blockquote><pre>
376      *      0 1 2 3 4 5 6 7 8 9 . -
377      * </pre></blockquote>
378      * <p>
379      * has the "numeric" attribute.
380      * <p>
381      * When the parser encounters a word token that has the format of a
382      * double precision floating-point number, it treats the token as a
383      * number rather than a word, by setting the <code>ttype</code>
384      * field to the value <code>TT_NUMBER</code> and putting the numeric
385      * value of the token into the <code>nval</code> field.
386      *
387      * @see     java.io.StreamTokenizer#nval
388      * @see     java.io.StreamTokenizer#TT_NUMBER
389      * @see     java.io.StreamTokenizer#ttype
390      */
391     public void parseNumbers() {
392     for (int i = '0'; i <= '9'; i++)
393         ctype[i] |= CT_DIGIT;
394     ctype['.'] |= CT_DIGIT;
395     ctype['-'] |= CT_DIGIT;
396     }
397 
398     /**
399      * Determines whether or not ends of line are treated as tokens.
400      * If the flag argument is true, this tokenizer treats end of lines
401      * as tokens; the <code>nextToken</code> method returns
402      * <code>TT_EOL</code> and also sets the <code>ttype</code> field to
403      * this value when an end of line is read.
404      * <p>
405      * A line is a sequence of characters ending with either a
406      * carriage-return character (<code>'&#92;r'</code>) or a newline
407      * character (<code>'&#92;n'</code>). In addition, a carriage-return
408      * character followed immediately by a newline character is treated
409      * as a single end-of-line token.
410      * <p>
411      * If the <code>flag</code> is false, end-of-line characters are
412      * treated as white space and serve only to separate tokens.
413      *
414      * @param   flag   <code>true</code> indicates that end-of-line characters
415      *                 are separate tokens; <code>false</code> indicates that
416      *                 end-of-line characters are white space.
417      * @see     java.io.StreamTokenizer#nextToken()
418      * @see     java.io.StreamTokenizer#ttype
419      * @see     java.io.StreamTokenizer#TT_EOL
420      */
421     public void eolIsSignificant(boolean flag) {
422     eolIsSignificantP = flag;
423     }
424 
425     /**
426      * Determines whether or not the tokenizer recognizes C-style comments.
427      * If the flag argument is <code>true</code>, this stream tokenizer
428      * recognizes C-style comments. All text between successive
429      * occurrences of <code>/*</code> and <code>*&#47;</code> are discarded.
430      * <p>
431      * If the flag argument is <code>false</code>, then C-style comments
432      * are not treated specially.
433      *
434      * @param   flag   <code>true</code> indicates to recognize and ignore
435      *                 C-style comments.
436      */
437     public void slashStarComments(boolean flag) {
438     slashStarCommentsP = flag;
439     }
440 
441     /**
442      * Determines whether or not the tokenizer recognizes C++-style comments.
443      * If the flag argument is <code>true</code>, this stream tokenizer
444      * recognizes C++-style comments. Any occurrence of two consecutive
445      * slash characters (<code>'/'</code>) is treated as the beginning of
446      * a comment that extends to the end of the line.
447      * <p>
448      * If the flag argument is <code>false</code>, then C++-style
449      * comments are not treated specially.
450      *
451      * @param   flag   <code>true</code> indicates to recognize and ignore
452      *                 C++-style comments.
453      */
454     public void slashSlashComments(boolean flag) {
455     slashSlashCommentsP = flag;
456     }
457 
458     /**
459      * Determines whether or not word token are automatically lowercased.
460      * If the flag argument is <code>true</code>, then the value in the
461      * <code>sval</code> field is lowercased whenever a word token is
462      * returned (the <code>ttype</code> field has the
463      * value <code>TT_WORD</code> by the <code>nextToken</code> method
464      * of this tokenizer.
465      * <p>
466      * If the flag argument is <code>false</code>, then the
467      * <code>sval</code> field is not modified.
468      *
469      * @param   fl   <code>true</code> indicates that all word tokens should
470      *               be lowercased.
471      * @see     java.io.StreamTokenizer#nextToken()
472      * @see     java.io.StreamTokenizer#ttype
473      * @see     java.io.StreamTokenizer#TT_WORD
474      */
475     public void lowerCaseMode(boolean fl) {
476     forceLower = fl;
477     }
478 
479     /** Read the next character */
480     private int read() throws IOException   {
481     if (reader != null)
482         return reader.read();
483     else if (input != null)
484         return input.read();
485     else
486         throw new IllegalStateException  ();
487     }
488 
489     /**
490      * Parses the next token from the input stream of this tokenizer.
491      * The type of the next token is returned in the <code>ttype</code>
492      * field. Additional information about the token may be in the
493      * <code>nval</code> field or the <code>sval</code> field of this
494      * tokenizer.
495      * <p>
496      * Typical clients of this
497      * class first set up the syntax tables and then sit in a loop
498      * calling nextToken to parse successive tokens until TT_EOF
499      * is returned.
500      *
501      * @return     the value of the <code>ttype</code> field.
502      * @exception  IOException  if an I/O error occurs.
503      * @see        java.io.StreamTokenizer#nval
504      * @see        java.io.StreamTokenizer#sval
505      * @see        java.io.StreamTokenizer#ttype
506      */
507     public int nextToken() throws IOException   {
508     if (pushedBack) {
509         pushedBack = false;
510         return ttype;
511     }
512     byte ct[] = ctype;
513     sval = null;
514 
515     int c = peekc;
516     if (c < 0)
517         c = NEED_CHAR;
518     if (c == SKIP_LF) {
519         c = read();
520         if (c < 0)
521         return ttype = TT_EOF;
522         if (c == '\n')
523         c = NEED_CHAR;
524     }
525     if (c == NEED_CHAR) {
526         c = read();
527         if (c < 0)
528         return ttype = TT_EOF;
529     }
530     ttype = c;      /* Just to be safe */
531 
532     /* Set peekc so that the next invocation of nextToken will read
533      * another character unless peekc is reset in this invocation
534      */
535     peekc = NEED_CHAR;
536 
537     int ctype = c < 256 ? ct[c] : CT_ALPHA;
538     while ((ctype & CT_WHITESPACE) != 0) {
539         if (c == '\r') {
540         LINENO++;
541         if (eolIsSignificantP) {
542             peekc = SKIP_LF;
543             return ttype = TT_EOL;
544         }
545         c = read();
546         if (c == '\n')
547             c = read();
548         } else {
549         if (c == '\n') {
550             LINENO++;
551             if (eolIsSignificantP) {
552             return ttype = TT_EOL;
553             }
554         }
555         c = read();
556         }
557         if (c < 0)
558         return ttype = TT_EOF;
559         ctype = c < 256 ? ct[c] : CT_ALPHA;
560     }
561 
562     if ((ctype & CT_DIGIT) != 0) {
563         boolean neg = false;
564         if (c == '-') {
565         c = read();
566         if (c != '.' && (c < '0' || c > '9')) {
567             peekc = c;
568             return ttype = '-';
569         }
570         neg = true;
571         }
572         double v = 0;
573         int decexp = 0;
574         int seendot = 0;
575         while (true) {
576         if (c == '.' && seendot == 0)
577             seendot = 1;
578         else if ('0' <= c && c <= '9') {
579             v = v * 10 + (c - '0');
580             decexp += seendot;
581         } else
582             break;
583         c = read();
584         }
585         peekc = c;
586         if (decexp != 0) {
587         double denom = 10;
588         decexp--;
589         while (decexp > 0) {
590             denom *= 10;
591             decexp--;
592         }
593         /* Do one division of a likely-to-be-more-accurate number */
594         v = v / denom;
595         }
596         nval = neg ? -v : v;
597         return ttype = TT_NUMBER;
598     }
599 
600     if ((ctype & CT_ALPHA) != 0) {
601         int i = 0;
602         do {
603         if (i >= buf.length) {
604             char nb[] = new char[buf.length * 2];
605             System.arraycopy(buf, 0, nb, 0, buf.length);
606             buf = nb;
607         }
608         buf[i++] = (char) c;
609         c = read();
610         ctype = c < 0 ? CT_WHITESPACE : c < 256 ? ct[c] : CT_ALPHA;
611         } while ((ctype & (CT_ALPHA | CT_DIGIT)) != 0);
612         peekc = c;
613         sval = String.copyValueOf(buf, 0, i);
614         if (forceLower)
615         sval = sval.toLowerCase();
616         return ttype = TT_WORD;
617     }
618 
619     if ((ctype & CT_QUOTE) != 0) {
620         ttype = c;
621         int i = 0;
622         /* Invariants (because \Octal needs a lookahead):
623          *   (i)  c contains char value
624          *   (ii) d contains the lookahead
625          */
626         int d = read();
627         while (d >= 0 && d != ttype && d != '\n' && d != '\r') {
628             if (d == '\\') {
629             c = read();
630             int first = c;   /* To allow \377, but not \477 */
631             if (c >= '0' && c <= '7') {
632             c = c - '0';
633             int c2 = read();
634             if ('0' <= c2 && c2 <= '7') {
635                 c = (c << 3) + (c2 - '0');
636                 c2 = read();
637                 if ('0' <= c2 && c2 <= '7' && first <= '3') {
638                 c = (c << 3) + (c2 - '0');
639                 d = read();
640                 } else
641                 d = c2;
642             } else
643               d = c2;
644             } else {
645                 switch (c) {
646             case 'a':
647                 c = 0x7;
648                 break;
649             case 'b':
650                 c = '\b';
651                 break;
652             case 'f':
653                 c = 0xC;
654                 break;
655             case 'n':
656                 c = '\n';
657                 break;
658                 case 'r':
659                 c = '\r';
660                 break;
661             case 't':
662                 c = '\t';
663                 break;
664             case 'v':
665                 c = 0xB;
666                 break;
667             }
668             d = read();
669             }
670         } else {
671             c = d;
672             d = read();
673         }
674         if (i >= buf.length) {
675             char nb[] = new char[buf.length * 2];
676             System.arraycopy(buf, 0, nb, 0, buf.length);
677             buf = nb;
678         }
679         buf[i++] = (char)c;
680         }
681 
682         /* If we broke out of the loop because we found a matching quote
683          * character then arrange to read a new character next time
684          * around; otherwise, save the character.
685          */
686         peekc = (d == ttype) ? NEED_CHAR : d;
687 
688         sval = String.copyValueOf(buf, 0, i);
689         return ttype;
690     }
691 
692     if (c == '/' && (slashSlashCommentsP || slashStarCommentsP)) {
693         c = read();
694         if (c == '*' && slashStarCommentsP) {
695         int prevc = 0;
696         while ((c = read()) != '/' || prevc != '*') {
697             if (c == '\r') {
698             LINENO++;
699             c = read();
700             if (c == '\n') {
701                 c = read();
702             }
703             } else {
704                 if (c == '\n') {
705                 LINENO++;
706                 c = read();
707             }
708             }
709             if (c < 0)
710                 return ttype = TT_EOF;
711             prevc = c;
712         }
713         return nextToken();
714         } else if (c == '/' && slashSlashCommentsP) {
715             while ((c = read()) != '\n' && c != '\r' && c >= 0);
716             peekc = c;
717         return nextToken();
718         } else {
719                 /* Now see if it is still a single line comment */
720                 if ((ct['/'] & CT_COMMENT) != 0) {
721                     while ((c = read()) != '\n' && c != '\r' && c >= 0);
722                     peekc = c;
723                     return nextToken();
724                 } else {
725                     peekc = c;
726                     return ttype = '/';
727                 }
728         }
729         }
730 
731         if ((ctype & CT_COMMENT) != 0) {
732             while ((c = read()) != '\n' && c != '\r' && c >= 0);
733             peekc = c;
734             return nextToken();
735         }
736 
737     return ttype = c;
738     }
739 
740     /**
741      * Causes the next call to the <code>nextToken</code> method of this
742      * tokenizer to return the current value in the <code>ttype</code>
743      * field, and not to modify the value in the <code>nval</code> or
744      * <code>sval</code> field.
745      *
746      * @see     java.io.StreamTokenizer#nextToken()
747      * @see     java.io.StreamTokenizer#nval
748      * @see     java.io.StreamTokenizer#sval
749      * @see     java.io.StreamTokenizer#ttype
750      */
751     public void pushBack() {
752         if (ttype != TT_NOTHING)   /* No-op if nextToken() not called */
753         pushedBack = true;
754     }
755 
756     /**
757      * Return the current line number.
758      *
759      * @return  the current line number of this stream tokenizer.
760      */
761     public int lineno() {
762     return LINENO;
763     }
764 
765     /**
766      * Returns the string representation of the current stream token and
767      * the line number it occurs on.
768      *
769      * <p>The precise string returned is unspecified, although the following
770      * example can be considered typical:
771      *
772      * <blockquote><pre>Token['a'], line 10</pre></blockquote>
773      *
774      * @return  a string representation of the token
775      * @see     java.io.StreamTokenizer#nval
776      * @see     java.io.StreamTokenizer#sval
777      * @see     java.io.StreamTokenizer#ttype
778      */
779     public String   toString() {
780     String   ret;
781     switch (ttype) {
782       case TT_EOF:
783         ret = "EOF";
784         break;
785       case TT_EOL:
786         ret = "EOL";
787         break;
788       case TT_WORD:
789         ret = sval;
790         break;
791       case TT_NUMBER:
792         ret = "n=" + nval;
793         break;
794       case TT_NOTHING:
795         ret = "NOTHING";
796         break;
797       default: {
798         /* 
799          * ttype is the first character of either a quoted string or
800          * is an ordinary character. ttype can definitely not be less
801          * than 0, since those are reserved values used in the previous
802          * case statements
803          */
804         if (ttype < 256 && 
805             ((ctype[ttype] & CT_QUOTE) != 0)) {
806             ret = sval;
807             break;
808         }
809 
810         char s[] = new char[3];
811         s[0] = s[2] = '\'';
812         s[1] = (char) ttype;
813         ret = new String  (s);
814         break;
815         }
816     }
817     return "Token[" + ret + "], line " + LINENO;
818     }
819 
820 }
821
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags