1 5 package org.h2.test.coverage; 6 7 import java.io.EOFException ; 8 import java.io.IOException ; 9 import java.io.Reader ; 10 11 14 public class Tokenizer { 15 private StringBuffer buffer; 16 17 private Reader reader; 18 19 private char[] buf = new char[20]; 20 private int peekc; 21 private int line = 1; 22 23 private byte[] charTypes = new byte[256]; 24 private static final byte WHITESPACE = 1; 25 private static final byte ALPHA = 4; 26 private static final byte QUOTE = 8; 27 28 private int type = TYPE_NOTHING; 29 public static final int TYPE_EOF = -1; 30 public static final int TYPE_WORD = -2; 31 private static final int TYPE_NOTHING = -3; 32 private String value; 33 34 private Tokenizer() { 35 wordChars('a', 'z'); 36 wordChars('A', 'Z'); 37 wordChars('0', '9'); 38 wordChars('.', '.'); 39 wordChars('+', '+'); 40 wordChars('-', '-'); 41 wordChars('_', '_'); 42 wordChars(128 + 32, 255); 43 whitespaceChars(0, ' '); 44 charTypes['"'] = QUOTE; 45 charTypes['\''] = QUOTE; 46 } 47 48 public Tokenizer(Reader r) { 49 this(); 50 reader = r; 51 } 52 53 public String getString() { 54 return value; 55 } 56 57 private void wordChars(int low, int hi) { 58 while (low <= hi) { 59 charTypes[low++] |= ALPHA; 60 } 61 } 62 63 private void whitespaceChars(int low, int hi) { 64 while (low <= hi) { 65 charTypes[low++] = WHITESPACE; 66 } 67 } 68 69 private int read() throws IOException { 70 int i = reader.read(); 71 if (i != -1) { 72 append(i); 73 } 74 return i; 75 } 76 77 public void initToken() { 78 buffer = new StringBuffer (); 79 } 80 81 public String getToken() { 82 buffer.setLength(buffer.length() - 1); 83 return buffer.toString(); 84 } 85 86 private void append(int i) { 87 buffer.append((char) i); 88 } 89 90 public int nextToken() throws IOException { 91 byte[] ct = charTypes; 92 int c; 93 value = null; 94 95 if (type == TYPE_NOTHING) { 96 c = read(); 97 if (c >= 0) { 98 type = c; 99 } 100 } else { 101 c = peekc; 102 if (c < 0) { 103 try { 104 c = read(); 105 if (c >= 0) { 106 type = c; 107 } 108 } catch (EOFException e) { 109 c = -1; 110 } 111 } 112 } 113 114 if (c < 0) { 115 return type = TYPE_EOF; 116 } 117 int ctype = c < 256 ? ct[c] : ALPHA; 118 while ((ctype & WHITESPACE) != 0) { 119 if (c == '\r') { 120 line++; 121 c = read(); 122 if (c == '\n') { 123 c = read(); 124 } 125 } else { 126 if (c == '\n') { 127 line++; 128 } 129 c = read(); 130 } 131 if (c < 0) { 132 return type = TYPE_EOF; 133 } 134 ctype = c < 256 ? ct[c] : ALPHA; 135 } 136 if ((ctype & ALPHA) != 0) { 137 initToken(); 138 append(c); 139 int i = 0; 140 do { 141 if (i >= buf.length) { 142 char[] nb = new char[buf.length * 2]; 143 System.arraycopy(buf, 0, nb, 0, buf.length); 144 buf = nb; 145 } 146 buf[i++] = (char) c; 147 c = read(); 148 ctype = c < 0 ? WHITESPACE : c < 256 ? ct[c] : ALPHA; 149 } while ((ctype & ALPHA) != 0); 150 peekc = c; 151 value = String.copyValueOf(buf, 0, i); 152 return type = TYPE_WORD; 153 } 154 if ((ctype & QUOTE) != 0) { 155 initToken(); 156 append(c); 157 type = c; 158 int i = 0; 159 peekc = read(); 161 while (peekc >= 0 && peekc != type && peekc != '\n' 162 && peekc != '\r') { 163 if (peekc == '\\') { 164 c = read(); 165 int first = c; if (c >= '0' && c <= '7') { 167 c = c - '0'; 168 int c2 = read(); 169 if ('0' <= c2 && c2 <= '7') { 170 c = (c << 3) + (c2 - '0'); 171 c2 = read(); 172 if ('0' <= c2 && c2 <= '7' && first <= '3') { 173 c = (c << 3) + (c2 - '0'); 174 peekc = read(); 175 } else { 176 peekc = c2; 177 } 178 } else { 179 peekc = c2; 180 } 181 } else { 182 switch (c) { 183 case 'b': 184 c = '\b'; 185 break; 186 case 'f': 187 c = '\f'; 188 break; 189 case 'n': 190 c = '\n'; 191 break; 192 case 'r': 193 c = '\r'; 194 break; 195 case 't': 196 c = '\t'; 197 break; 198 } 199 peekc = read(); 200 } 201 } else { 202 c = peekc; 203 peekc = read(); 204 } 205 206 if (i >= buf.length) { 207 char[] nb = new char[buf.length * 2]; 208 System.arraycopy(buf, 0, nb, 0, buf.length); 209 buf = nb; 210 } 211 buf[i++] = (char) c; 212 } 213 if (peekc == type) { 214 peekc = read(); 216 } 217 value = String.copyValueOf(buf, 0, i); 218 return type; 219 } 220 if (c == '/') { 221 c = read(); 222 if (c == '*') { 223 int prevc = 0; 224 while ((c = read()) != '/' || prevc != '*') { 225 if (c == '\r') { 226 line++; 227 c = read(); 228 if (c == '\n') { 229 c = read(); 230 } 231 } else { 232 if (c == '\n') { 233 line++; 234 c = read(); 235 } 236 } 237 if (c < 0) { 238 return type = TYPE_EOF; 239 } 240 prevc = c; 241 } 242 peekc = read(); 243 return nextToken(); 244 } else if (c == '/') { 245 while ((c = read()) != '\n' && c != '\r' && c >= 0) { 246 } 248 peekc = c; 249 return nextToken(); 250 } else { 251 peekc = c; 252 return type = '/'; 253 } 254 } 255 peekc = read(); 256 return type = c; 257 } 258 259 public int getLine() { 260 return line; 261 } 262 } 263 264 | Popular Tags |