KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > lateralnz > c3d > util > Tokenizer


1 /* ====================================================================
2  * The LateralNZ Software License, Version 1.0
3  *
4  * Copyright (c) 2003 LateralNZ. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  *
10  * 1. Redistributions of source code must retain the above copyright
11  * notice, this list of conditions and the following disclaimer.
12  *
13  * 2. Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  *
18  * 3. The end-user documentation included with the redistribution,
19  * if any, must include the following acknowledgment:
20  * "This product includes software developed by
21  * LateralNZ (http://www.lateralnz.org/) and other third parties."
22  * Alternately, this acknowledgment may appear in the software itself,
23  * if and wherever such third-party acknowledgments normally appear.
24  *
25  * 4. The names "LateralNZ" must not be used to endorse or promote
26  * products derived from this software without prior written
27  * permission. For written permission, please
28  * contact oss@lateralnz.org.
29  *
30  * 5. Products derived from this software may not be called "Panther",
31  * or "Lateral" or "LateralNZ", nor may "PANTHER" or "LATERAL" or
32  * "LATERALNZ" appear in their name, without prior written
33  * permission of LateralNZ.
34  *
35  * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
36  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
37  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
38  * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
39  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
42  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
43  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
44  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
45  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
46  * SUCH DAMAGE.
47  * ====================================================================
48  *
49  * This software consists of voluntary contributions made by many
50  * individuals on behalf of LateralNZ. For more
51  * information on Lateral, please see http://www.lateralnz.com/ or
52  * http://www.lateralnz.org
53  *
54  */

55 package org.lateralnz.c3d.util;
56
57 /**
58  * a custom string tokenizer (based on the standard Java StringTokenizer)
59  * that supports marking a position and returning to it. This is most
60  * useful in lexical analysis/parsing/etc.
61  */

62 public class Tokenizer {
63   private static final String JavaDoc EMPTY = "";
64
65   private String JavaDoc delimiters;
66   private boolean rtnDelims;
67   private String JavaDoc returnDelimiters;
68
69   private char chars[];
70
71   int currentPos;
72
73   private int maxPos;
74
75   int[] mark = new int[100];
76   int markpos = 0;
77
78   public Tokenizer(String JavaDoc text, String JavaDoc delimiters, String JavaDoc returnDelimiters) {
79     setString(text);
80     this.delimiters = delimiters;
81     this.returnDelimiters = returnDelimiters;
82     this.rtnDelims = (returnDelimiters != null && !returnDelimiters.equals(EMPTY));
83   }
84
85  /**
86   * set the string to tokenize (a helper to save having to create
87   * this object every time)
88   */

89   public final void setString(String JavaDoc text) {
90     this.maxPos = text.length();
91     this.currentPos = 0;
92
93     this.chars = new char[maxPos];
94     text.getChars(0,maxPos,this.chars,0);
95   }
96
97  /**
98   * clean up and shutdown
99   */

100   public final void destroy() {
101     chars = null;
102   }
103
104  /**
105   * mark the current position
106   */

107   public final void mark() {
108     markpos++;
109     if (markpos >= mark.length) {
110       int[] newmark = new int[mark.length * 2];
111       System.arraycopy(mark, 0, newmark, 0, mark.length);
112       mark = newmark;
113     }
114     mark[markpos] = currentPos;
115   }
116
117  /**
118   * return to the last marked position
119   */

120   public final void release() {
121     currentPos = mark[markpos];
122     markpos--;
123   }
124
125  /**
126   * reset the position counter to start tokenizing at the beginning again
127   */

128   public final void reset() {
129     currentPos = 0;
130   }
131
132  /**
133   * return true if there are more tokens
134   */

135   public final boolean hasMoreTokens() {
136     boolean rtn = false;
137     mark();
138     String JavaDoc s = nextToken();
139     if (s != null && !s.equals(EMPTY)) {
140       rtn = true;
141     }
142     release();
143
144     return rtn;
145   }
146
147  /**
148   * get the next token
149   */

150   public final String JavaDoc nextToken() {
151     StringBuffer JavaDoc rtn = new StringBuffer JavaDoc();
152
153     while (currentPos < maxPos && delimiters.indexOf(chars[currentPos]) >= 0) {
154       if (rtnDelims && returnDelimiters.indexOf(chars[currentPos]) >= 0) {
155         rtn.append(chars[currentPos]);
156         currentPos++;
157         return rtn.toString();
158       }
159       currentPos++;
160     }
161
162     while (currentPos < maxPos && delimiters.indexOf(chars[currentPos]) < 0) {
163       rtn.append(chars[currentPos]);
164       currentPos++;
165     }
166
167     /*
168     while (currentPos < maxPos && delimiters.indexOf(chars[currentPos]) >= 0) {
169       currentPos++;
170     }
171
172     while (currentPos < maxPos && delimiters.indexOf(chars[currentPos]) < 0) {
173       rtn.append(chars[currentPos]);
174       currentPos++;
175     }
176 */

177     return rtn.toString();
178   }
179
180  /**
181   * look at the next token without moving the position indicator.
182   * (basically the works out to mark, next token, release)
183   */

184   public final String JavaDoc peekNextToken() {
185     mark();
186     String JavaDoc rtn = nextToken();
187     release();
188     return rtn;
189   }
190
191 }
Popular Tags