KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > h2 > test > coverage > Tokenizer


1 /*
2  * Copyright 2004-2006 H2 Group. Licensed under the H2 License, Version 1.0 (http://h2database.com/html/license.html).
3  * Initial Developer: H2 Group
4  */

5 package org.h2.test.coverage;
6
7 import java.io.EOFException JavaDoc;
8 import java.io.IOException JavaDoc;
9 import java.io.Reader JavaDoc;
10
11 /**
12  * Helper class for the java file parser.
13  */

14 public class Tokenizer {
15     private StringBuffer JavaDoc buffer;
16
17     private Reader JavaDoc reader;
18
19     private char[] buf = new char[20];
20     private int peekc;
21     private int line = 1;
22
23     private byte[] charTypes = new byte[256];
24     private static final byte WHITESPACE = 1;
25     private static final byte ALPHA = 4;
26     private static final byte QUOTE = 8;
27
28     private int type = TYPE_NOTHING;
29     public static final int TYPE_EOF = -1;
30     public static final int TYPE_WORD = -2;
31     private static final int TYPE_NOTHING = -3;
32     private String JavaDoc value;
33
34     private Tokenizer() {
35         wordChars('a', 'z');
36         wordChars('A', 'Z');
37         wordChars('0', '9');
38         wordChars('.', '.');
39         wordChars('+', '+');
40         wordChars('-', '-');
41         wordChars('_', '_');
42         wordChars(128 + 32, 255);
43         whitespaceChars(0, ' ');
44         charTypes['"'] = QUOTE;
45         charTypes['\''] = QUOTE;
46     }
47
48     public Tokenizer(Reader JavaDoc r) {
49         this();
50         reader = r;
51     }
52
53     public String JavaDoc getString() {
54         return value;
55     }
56
57     private void wordChars(int low, int hi) {
58         while (low <= hi) {
59             charTypes[low++] |= ALPHA;
60         }
61     }
62
63     private void whitespaceChars(int low, int hi) {
64         while (low <= hi) {
65             charTypes[low++] = WHITESPACE;
66         }
67     }
68
69     private int read() throws IOException JavaDoc {
70         int i = reader.read();
71         if (i != -1) {
72             append(i);
73         }
74         return i;
75     }
76
77     public void initToken() {
78         buffer = new StringBuffer JavaDoc();
79     }
80
81     public String JavaDoc getToken() {
82         buffer.setLength(buffer.length() - 1);
83         return buffer.toString();
84     }
85
86     private void append(int i) {
87         buffer.append((char) i);
88     }
89
90     public int nextToken() throws IOException JavaDoc {
91         byte[] ct = charTypes;
92         int c;
93         value = null;
94
95         if (type == TYPE_NOTHING) {
96             c = read();
97             if (c >= 0) {
98                 type = c;
99             }
100         } else {
101             c = peekc;
102             if (c < 0) {
103                 try {
104                     c = read();
105                     if (c >= 0) {
106                         type = c;
107                     }
108                 } catch (EOFException JavaDoc e) {
109                     c = -1;
110                 }
111             }
112         }
113
114         if (c < 0) {
115             return type = TYPE_EOF;
116         }
117         int ctype = c < 256 ? ct[c] : ALPHA;
118         while ((ctype & WHITESPACE) != 0) {
119             if (c == '\r') {
120                 line++;
121                 c = read();
122                 if (c == '\n') {
123                     c = read();
124                 }
125             } else {
126                 if (c == '\n') {
127                     line++;
128                 }
129                 c = read();
130             }
131             if (c < 0) {
132                 return type = TYPE_EOF;
133             }
134             ctype = c < 256 ? ct[c] : ALPHA;
135         }
136         if ((ctype & ALPHA) != 0) {
137             initToken();
138             append(c);
139             int i = 0;
140             do {
141                 if (i >= buf.length) {
142                     char[] nb = new char[buf.length * 2];
143                     System.arraycopy(buf, 0, nb, 0, buf.length);
144                     buf = nb;
145                 }
146                 buf[i++] = (char) c;
147                 c = read();
148                 ctype = c < 0 ? WHITESPACE : c < 256 ? ct[c] : ALPHA;
149             } while ((ctype & ALPHA) != 0);
150             peekc = c;
151             value = String.copyValueOf(buf, 0, i);
152             return type = TYPE_WORD;
153         }
154         if ((ctype & QUOTE) != 0) {
155             initToken();
156             append(c);
157             type = c;
158             int i = 0;
159             // \octal needs a lookahead
160
peekc = read();
161             while (peekc >= 0 && peekc != type && peekc != '\n'
162                     && peekc != '\r') {
163                 if (peekc == '\\') {
164                     c = read();
165                     int first = c; // to allow \377, but not \477
166
if (c >= '0' && c <= '7') {
167                         c = c - '0';
168                         int c2 = read();
169                         if ('0' <= c2 && c2 <= '7') {
170                             c = (c << 3) + (c2 - '0');
171                             c2 = read();
172                             if ('0' <= c2 && c2 <= '7' && first <= '3') {
173                                 c = (c << 3) + (c2 - '0');
174                                 peekc = read();
175                             } else {
176                                 peekc = c2;
177                             }
178                         } else {
179                             peekc = c2;
180                         }
181                     } else {
182                         switch (c) {
183                         case 'b':
184                             c = '\b';
185                             break;
186                         case 'f':
187                             c = '\f';
188                             break;
189                         case 'n':
190                             c = '\n';
191                             break;
192                         case 'r':
193                             c = '\r';
194                             break;
195                         case 't':
196                             c = '\t';
197                             break;
198                         }
199                         peekc = read();
200                     }
201                 } else {
202                     c = peekc;
203                     peekc = read();
204                 }
205
206                 if (i >= buf.length) {
207                     char[] nb = new char[buf.length * 2];
208                     System.arraycopy(buf, 0, nb, 0, buf.length);
209                     buf = nb;
210                 }
211                 buf[i++] = (char) c;
212             }
213             if (peekc == type) {
214                 // keep \n or \r intact in peekc
215
peekc = read();
216             }
217             value = String.copyValueOf(buf, 0, i);
218             return type;
219         }
220         if (c == '/') {
221             c = read();
222             if (c == '*') {
223                 int prevc = 0;
224                 while ((c = read()) != '/' || prevc != '*') {
225                     if (c == '\r') {
226                         line++;
227                         c = read();
228                         if (c == '\n') {
229                             c = read();
230                         }
231                     } else {
232                         if (c == '\n') {
233                             line++;
234                             c = read();
235                         }
236                     }
237                     if (c < 0) {
238                         return type = TYPE_EOF;
239                     }
240                     prevc = c;
241                 }
242                 peekc = read();
243                 return nextToken();
244             } else if (c == '/') {
245                 while ((c = read()) != '\n' && c != '\r' && c >= 0) {
246                     // nothing
247
}
248                 peekc = c;
249                 return nextToken();
250             } else {
251                 peekc = c;
252                 return type = '/';
253             }
254         }
255         peekc = read();
256         return type = c;
257     }
258
259     public int getLine() {
260         return line;
261     }
262 }
263
264
Popular Tags