StringTokenizer


1   /*
2    * @(#)StringTokenizer.java 1.34 04/05/05
3    *
4    * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
5    * SUN PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
6    */
7   
8   package java.util;
9   
10  import java.lang.*;
11  
12  /**
13   * The string tokenizer class allows an application to break a 
14   * string into tokens. The tokenization method is much simpler than 
15   * the one used by the <code>StreamTokenizer</code> class. The 
16   * <code>StringTokenizer</code> methods do not distinguish among 
17   * identifiers, numbers, and quoted strings, nor do they recognize 
18   * and skip comments. 
19   * <p>
20   * The set of delimiters (the characters that separate tokens) may 
21   * be specified either at creation time or on a per-token basis. 
22   * <p>
23   * An instance of <code>StringTokenizer</code> behaves in one of two 
24   * ways, depending on whether it was created with the 
25   * <code>returnDelims</code> flag having the value <code>true</code> 
26   * or <code>false</code>: 
27   * <ul>
28   * <li>If the flag is <code>false</code>, delimiter characters serve to 
29   *     separate tokens. A token is a maximal sequence of consecutive 
30   *     characters that are not delimiters. 
31   * <li>If the flag is <code>true</code>, delimiter characters are themselves 
32   *     considered to be tokens. A token is thus either one delimiter 
33   *     character, or a maximal sequence of consecutive characters that are 
34   *     not delimiters.
35   * </ul><p>
36   * A <tt>StringTokenizer</tt> object internally maintains a current 
37   * position within the string to be tokenized. Some operations advance this 
38   * current position past the characters processed.<p>
39   * A token is returned by taking a substring of the string that was used to 
40   * create the <tt>StringTokenizer</tt> object.
41   * <p>
42   * The following is one example of the use of the tokenizer. The code:
43   * <blockquote><pre>
44   *     StringTokenizer st = new StringTokenizer("this is a test");
45   *     while (st.hasMoreTokens()) {
46   *         System.out.println(st.nextToken());
47   *     }
48   * </pre></blockquote>
49   * <p>
50   * prints the following output:
51   * <blockquote><pre>
52   *     this
53   *     is
54   *     a
55   *     test
56   * </pre></blockquote>
57   *
58   * <p>
59   * <tt>StringTokenizer</tt> is a legacy class that is retained for
60   * compatibility reasons although its use is discouraged in new code. It is
61   * recommended that anyone seeking this functionality use the <tt>split</tt>
62   * method of <tt>String</tt> or the java.util.regex package instead.
63   * <p>
64   * The following example illustrates how the <tt>String.split</tt>
65   * method can be used to break up a string into its basic tokens:
66   * <blockquote><pre>
67   *     String[] result = "this is a test".split("\\s");
68   *     for (int x=0; x&lt;result.length; x++)
69   *         System.out.println(result[x]);
70   * </pre></blockquote>
71   * <p>
72   * prints the following output:
73   * <blockquote><pre>
74   *     this
75   *     is
76   *     a
77   *     test
78   * </pre></blockquote>
79   *
80   * @author  unascribed
81   * @version 1.34, 05/05/04
82   * @see     java.io.StreamTokenizer
83   * @since   JDK1.0
84   */
85  public
86  class StringTokenizer implements Enumeration  <Object  > {
87      private int currentPosition;
88      private int newPosition;
89      private int maxPosition;
90      private String   str;
91      private String   delimiters;
92      private boolean retDelims;
93      private boolean delimsChanged;
94  
95      /**
96       * maxDelimCodePoint stores the value of the delimiter character with the
97       * highest value. It is used to optimize the detection of delimiter
98       * characters.
99       *
100      * It is unlikely to provide any optimization benefit in the
101      * hasSurrogates case because most string characters will be
102      * smaller than the limit, but we keep it so that the two code
103      * paths remain similar.
104      */
105     private int maxDelimCodePoint;
106 
107     /**
108      * If delimiters include any surrogates (including surrogate
109      * pairs), hasSurrogates is true and the tokenizer uses the
110      * different code path. This is because String.indexOf(int)
111      * doesn't handle unpaired surrogates as a single character.
112      */
113     private boolean hasSurrogates = false;
114 
115     /**
116      * When hasSurrogates is true, delimiters are converted to code
117      * points and isDelimiter(int) is used to determine if the given
118      * codepoint is a delimiter.
119      */
120     private int[] delimiterCodePoints;
121 
122     /**
123      * Set maxDelimCodePoint to the highest char in the delimiter set.
124      */
125     private void setMaxDelimCodePoint() {
126         if (delimiters == null) {
127             maxDelimCodePoint = 0;
128             return;
129         }
130 
131     int m = 0;
132     int c;
133     int count = 0;
134     for (int i = 0; i < delimiters.length(); i += Character.charCount(c)) {
135         c = delimiters.charAt(i);
136         if (c >= Character.MIN_HIGH_SURROGATE && c <= Character.MAX_LOW_SURROGATE) {
137         c = delimiters.codePointAt(i);
138         hasSurrogates = true;
139         }
140         if (m < c)
141         m = c;
142         count++;
143     }
144     maxDelimCodePoint = m;
145 
146     if (hasSurrogates) {
147         delimiterCodePoints = new int[count];
148         for (int i = 0, j = 0; i < count; i++, j += Character.charCount(c)) {
149         c = delimiters.codePointAt(j);
150         delimiterCodePoints[i] = c;
151         }
152     }
153     }
154 
155     /**
156      * Constructs a string tokenizer for the specified string. All  
157      * characters in the <code>delim</code> argument are the delimiters 
158      * for separating tokens. 
159      * <p>
160      * If the <code>returnDelims</code> flag is <code>true</code>, then 
161      * the delimiter characters are also returned as tokens. Each 
162      * delimiter is returned as a string of length one. If the flag is 
163      * <code>false</code>, the delimiter characters are skipped and only 
164      * serve as separators between tokens. 
165      * <p>
166      * Note that if <tt>delim</tt> is <tt>null</tt>, this constructor does
167      * not throw an exception. However, trying to invoke other methods on the
168      * resulting <tt>StringTokenizer</tt> may result in a 
169      * <tt>NullPointerException</tt>.
170      *
171      * @param   str            a string to be parsed.
172      * @param   delim          the delimiters.
173      * @param   returnDelims   flag indicating whether to return the delimiters
174      *                         as tokens.
175      * @exception NullPointerException if str is <CODE>null</CODE>
176      */
177     public StringTokenizer(String   str, String   delim, boolean returnDelims) {
178     currentPosition = 0;
179     newPosition = -1;
180     delimsChanged = false;
181     this.str = str;
182     maxPosition = str.length();
183     delimiters = delim;
184     retDelims = returnDelims;
185         setMaxDelimCodePoint();
186     }
187 
188     /**
189      * Constructs a string tokenizer for the specified string. The 
190      * characters in the <code>delim</code> argument are the delimiters 
191      * for separating tokens. Delimiter characters themselves will not 
192      * be treated as tokens.
193      * <p>
194      * Note that if <tt>delim</tt> is <tt>null</tt>, this constructor does
195      * not throw an exception. However, trying to invoke other methods on the
196      * resulting <tt>StringTokenizer</tt> may result in a
197      * <tt>NullPointerException</tt>.
198      *
199      * @param   str     a string to be parsed.
200      * @param   delim   the delimiters.
201      * @exception NullPointerException if str is <CODE>null</CODE>
202      */
203     public StringTokenizer(String   str, String   delim) {
204     this(str, delim, false);
205     }
206 
207     /**
208      * Constructs a string tokenizer for the specified string. The 
209      * tokenizer uses the default delimiter set, which is 
210      * <code>"&nbsp;&#92;t&#92;n&#92;r&#92;f"</code>: the space character, 
211      * the tab character, the newline character, the carriage-return character,
212      * and the form-feed character. Delimiter characters themselves will 
213      * not be treated as tokens.
214      *
215      * @param   str   a string to be parsed.
216      * @exception NullPointerException if str is <CODE>null</CODE> 
217      */
218     public StringTokenizer(String   str) {
219     this(str, " \t\n\r\f", false);
220     }
221 
222     /**
223      * Skips delimiters starting from the specified position. If retDelims
224      * is false, returns the index of the first non-delimiter character at or
225      * after startPos. If retDelims is true, startPos is returned.
226      */
227     private int skipDelimiters(int startPos) {
228         if (delimiters == null)
229             throw new NullPointerException  ();
230 
231         int position = startPos;
232     while (!retDelims && position < maxPosition) {
233         if (!hasSurrogates) {
234         char c = str.charAt(position);
235         if ((c > maxDelimCodePoint) || (delimiters.indexOf(c) < 0))
236             break;
237         position++;
238         } else {
239         int c = str.codePointAt(position);
240         if ((c > maxDelimCodePoint) || !isDelimiter(c)) {
241             break;
242         }
243         position += Character.charCount(c);
244         }
245     }
246         return position;
247     }
248 
249     /**
250      * Skips ahead from startPos and returns the index of the next delimiter
251      * character encountered, or maxPosition if no such delimiter is found.
252      */
253     private int scanToken(int startPos) {
254         int position = startPos;
255         while (position < maxPosition) {
256         if (!hasSurrogates) {
257         char c = str.charAt(position);
258         if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0))
259             break;
260         position++;
261         } else {
262         int c = str.codePointAt(position);
263         if ((c <= maxDelimCodePoint) && isDelimiter(c))
264             break;
265         position += Character.charCount(c);
266         }
267     }
268     if (retDelims && (startPos == position)) {
269         if (!hasSurrogates) {
270         char c = str.charAt(position);
271         if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0))
272             position++;
273         } else {
274         int c = str.codePointAt(position);
275         if ((c <= maxDelimCodePoint) && isDelimiter(c))
276             position += Character.charCount(c);
277         }
278         }
279         return position;
280     }
281 
282     private boolean isDelimiter(int codePoint) {
283     for (int i = 0; i < delimiterCodePoints.length; i++) {
284         if (delimiterCodePoints[i] == codePoint) {
285         return true;
286         }
287     }
288     return false;
289     }
290 
291     /**
292      * Tests if there are more tokens available from this tokenizer's string. 
293      * If this method returns <tt>true</tt>, then a subsequent call to 
294      * <tt>nextToken</tt> with no argument will successfully return a token.
295      *
296      * @return  <code>true</code> if and only if there is at least one token 
297      *          in the string after the current position; <code>false</code> 
298      *          otherwise.
299      */
300     public boolean hasMoreTokens() {
301     /*
302      * Temporarily store this position and use it in the following
303      * nextToken() method only if the delimiters haven't been changed in
304      * that nextToken() invocation.
305      */
306     newPosition = skipDelimiters(currentPosition);
307     return (newPosition < maxPosition);
308     }
309 
310     /**
311      * Returns the next token from this string tokenizer.
312      *
313      * @return     the next token from this string tokenizer.
314      * @exception  NoSuchElementException  if there are no more tokens in this
315      *               tokenizer's string.
316      */
317     public String   nextToken() {
318     /* 
319      * If next position already computed in hasMoreElements() and
320      * delimiters have changed between the computation and this invocation,
321      * then use the computed value.
322      */
323 
324     currentPosition = (newPosition >= 0 && !delimsChanged) ?  
325         newPosition : skipDelimiters(currentPosition);
326 
327     /* Reset these anyway */
328     delimsChanged = false;
329     newPosition = -1;
330 
331     if (currentPosition >= maxPosition)
332         throw new NoSuchElementException  ();
333     int start = currentPosition;
334     currentPosition = scanToken(currentPosition);
335     return str.substring(start, currentPosition);
336     }
337 
338     /**
339      * Returns the next token in this string tokenizer's string. First, 
340      * the set of characters considered to be delimiters by this 
341      * <tt>StringTokenizer</tt> object is changed to be the characters in 
342      * the string <tt>delim</tt>. Then the next token in the string
343      * after the current position is returned. The current position is 
344      * advanced beyond the recognized token.  The new delimiter set 
345      * remains the default after this call. 
346      *
347      * @param      delim   the new delimiters.
348      * @return     the next token, after switching to the new delimiter set.
349      * @exception  NoSuchElementException  if there are no more tokens in this
350      *               tokenizer's string.
351      * @exception NullPointerException if delim is <CODE>null</CODE>
352      */
353     public String   nextToken(String   delim) {
354     delimiters = delim;
355 
356     /* delimiter string specified, so set the appropriate flag. */
357     delimsChanged = true;
358 
359         setMaxDelimCodePoint();
360     return nextToken();
361     }
362 
363     /**
364      * Returns the same value as the <code>hasMoreTokens</code>
365      * method. It exists so that this class can implement the
366      * <code>Enumeration</code> interface. 
367      *
368      * @return  <code>true</code> if there are more tokens;
369      *          <code>false</code> otherwise.
370      * @see     java.util.Enumeration
371      * @see     java.util.StringTokenizer#hasMoreTokens()
372      */
373     public boolean hasMoreElements() {
374     return hasMoreTokens();
375     }
376 
377     /**
378      * Returns the same value as the <code>nextToken</code> method,
379      * except that its declared return value is <code>Object</code> rather than
380      * <code>String</code>. It exists so that this class can implement the
381      * <code>Enumeration</code> interface. 
382      *
383      * @return     the next token in the string.
384      * @exception  NoSuchElementException  if there are no more tokens in this
385      *               tokenizer's string.
386      * @see        java.util.Enumeration
387      * @see        java.util.StringTokenizer#nextToken()
388      */
389     public Object   nextElement() {
390     return nextToken();
391     }
392 
393     /**
394      * Calculates the number of times that this tokenizer's 
395      * <code>nextToken</code> method can be called before it generates an 
396      * exception. The current position is not advanced.
397      *
398      * @return  the number of tokens remaining in the string using the current
399      *          delimiter set.
400      * @see     java.util.StringTokenizer#nextToken()
401      */
402     public int countTokens() {
403     int count = 0;
404     int currpos = currentPosition;
405     while (currpos < maxPosition) {
406             currpos = skipDelimiters(currpos);
407         if (currpos >= maxPosition)
408         break;
409             currpos = scanToken(currpos);
410         count++;
411     }
412     return count;
413     }
414 }
415
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags