|
1
23
24
28
29
30
35
36 package com.sun.enterprise.admin.util;
37
38 import java.util.Iterator  ;
39
40 import junit.framework.TestCase;
41
42
43
44 public class TokenizerTest extends TestCase
45 {
46 public
47 TokenizerTest( String name )
48 {
49 super( name );
50 }
51
52 public final static char TAB = '\t';
53 public final static char BACKSLASH = '\\';
54 public final static char QUOTE_CHAR = '\"';
55 public final static String WHITE_SPACE = " " + TAB;
56 public final static char ESCAPE_CHAR = BACKSLASH;
57 public final static String ESCAPABLE_CHARS = "" + ESCAPE_CHAR + WHITE_SPACE + QUOTE_CHAR;
58
59 Tokenizer
60 create( String input, boolean multipleDelimsCountAsOne )
61 throws TokenizerException
62 {
63 return( new TokenizerImpl( input,
64 WHITE_SPACE, multipleDelimsCountAsOne, ESCAPE_CHAR, ESCAPABLE_CHARS) );
65 }
66
67 Tokenizer
68 create( String input )
69 throws TokenizerException
70 {
71 return( new TokenizerImpl( input,
72 WHITE_SPACE, false, ESCAPE_CHAR, ESCAPABLE_CHARS) );
73 }
74
75 public void
76 testEmpty()
77 throws TokenizerException
78 {
79 final Tokenizer tk = create( "" );
80
81 assertEquals( "expecting no tokens", 0, tk.getTokens().length );
82 }
83
84
85 public void
86 testLeadingDelim()
87 throws TokenizerException
88 {
89 final String input = WHITE_SPACE.charAt( 0 ) + "hello";
90 final String [] tokens = create( input, false ).getTokens();
91
92 assertEquals( 2, tokens.length );
93 assertEquals( "", tokens[ 0 ] );
94 assertEquals( "hello", tokens[ 1 ] );
95 }
96
97
98 public void
99 testSingleDelimOnly()
100 throws TokenizerException
101 {
102 final String input = "" + WHITE_SPACE.charAt( 0 );
103 final String [] tokens = create( input, false ).getTokens();
104
105 assertEquals( 2, tokens.length );
106 assertEquals( "", tokens[ 0 ] );
107 assertEquals( "", tokens[ 1 ] );
108 }
109
110 public void
111 testOnlyDelims()
112 throws TokenizerException
113 {
114 final String input = WHITE_SPACE + WHITE_SPACE + WHITE_SPACE;
115 final String [] tokens = create( input, false ).getTokens();
116
117 assertEquals( (WHITE_SPACE.length() * 3) + 1, tokens.length );
118 }
119
120 public void
121 testSingleToken()
122 throws TokenizerException
123 {
124 final String input = "hello";
125 final String [] tokens = create( input, false ).getTokens();
126
127 assertEquals( "expecting 1 token", 1, tokens.length );
128 assertEquals( "expecting " + input, input, tokens[ 0 ] );
129 }
130
131 public void
132 testMultipleTokens()
133 throws TokenizerException
134 {
135 final String input = "hello there 1 2 3 4 5";
136 final String [] tokens = create( input ).getTokens();
137
138 assertEquals( "expecting 7 tokens", 7, tokens.length );
139 assertEquals( tokens[ 0 ], "hello" );
140 assertEquals( tokens[ 1 ], "there" );
141 assertEquals( tokens[ 2 ], "1" );
142 assertEquals( tokens[ 3 ], "2" );
143 assertEquals( tokens[ 4 ], "3" );
144 assertEquals( tokens[ 5 ], "4" );
145 assertEquals( tokens[ 6 ], "5" );
146 }
147
148 public void
149 testWhiteSpaceEquality()
150 throws TokenizerException
151 {
152 final String input1 = "hello there 1 2 3 4 5";
153 final String input2 = "hello\tthere 1\t2 3 4\t5";
154 final String [] tokens1 = create( input1 ).getTokens();
155 final String [] tokens2 = create( input2 ).getTokens();
156
157
158 assertEquals( tokens1.length, tokens2.length );
159 for( int i = 0; i < tokens1.length; ++i )
160 {
161 assertEquals( tokens1[ i ], tokens2[ i ] );
162 }
163 }
164
165 public void
166 testEscapedNewlineCR()
167 throws TokenizerException
168 {
169 final String TEST = "test" + BACKSLASH + "n" + BACKSLASH + "r" + BACKSLASH + "t";
170 final String [] tokens = create( TEST ).getTokens();
171
172 assertEquals( 1, tokens.length );
173 assertEquals( "test\n\r\t", tokens[ 0 ] );
174 }
175
176 public void
177 testEscaping()
178 throws TokenizerException
179 {
180 final StringBuffer b = new StringBuffer ();
182 for( int i = 0; i < ESCAPABLE_CHARS.length(); ++i )
183 {
184 b.append( "\\" + ESCAPABLE_CHARS.charAt( i ) );
185 }
186 final String [] tokens = create( b.toString() ).getTokens();
187
188 assertEquals( "expecting 1 token", 1, tokens.length );
189 assertEquals( "expecting match", ESCAPABLE_CHARS, tokens[ 0 ] );
190 }
191
192
193 public void
194 testTrailingDelimiter1()
195 throws TokenizerException
196 {
197 final String input = "hello" + WHITE_SPACE.charAt( 0 );
198 final String [] tokens = create( input, false ).getTokens();
199
200 assertEquals( 2, tokens.length );
201 }
202
203 public void
204 testTrailingDelimiter2()
205 throws TokenizerException
206 {
207 final String input = "hello" + WHITE_SPACE.charAt( 0 ) +
208 "\"there\"" + WHITE_SPACE.charAt( 0 );
209 final String [] tokens = create( input, false ).getTokens();
210
211 assertEquals( 3, tokens.length );
212 }
213
214 public void
215 testMultipleDelimsWithNoData()
216 throws TokenizerException
217 {
218 final String input = "" + WHITE_SPACE.charAt( 0 );
219 final String [] tokens = create( input, false ).getTokens();
220
221 assertEquals( input.length() + 1, tokens.length );
222 }
223
224
225 public void
226 testMultipleDelimsAsOne()
227 throws TokenizerException
228 {
229 final String input = "HELLO" + WHITE_SPACE + "THERE" + WHITE_SPACE + WHITE_SPACE;
230 final String [] tokens = create( input, true ).getTokens();
231
232 assertEquals( 3, tokens.length );
233 }
234
235 static final char QUOTE = '\"';
236 static String
237 quote( String s )
238 {
239 return( QUOTE + s + QUOTE );
240 }
241
242
243 public void
244 testEmptyQuotedString()
245 throws TokenizerException
246 {
247 final String input = quote( "" );
248 final String [] tokens = create( input ).getTokens();
249
250 assertEquals( 1, tokens.length );
251 assertEquals( "", tokens[ 0 ] );
252 }
253 public void
254 testEmptyQuotedStrings()
255 throws TokenizerException
256 {
257 final String input = quote( "" ) + quote( "" ) + quote( "" ) + quote( "" );
258 final String [] tokens = create( input ).getTokens();
259
260 assertEquals( 1, tokens.length );
261 assertEquals( "", tokens[ 0 ] );
262 }
263
264 public void
265 testEmptyQuotedStringsSeparatedByDelim()
266 throws TokenizerException
267 {
268 final String input = quote( "" ) + WHITE_SPACE.charAt( 0 ) + quote( "" );
269 final String [] tokens = create( input ).getTokens();
270
271 assertEquals( 2, tokens.length );
272 }
273
274 public void
275 testAdjacentQuotedStrings()
276 throws TokenizerException
277 {
278 final String input = quote( "xxx" ) + quote( "yyy" ) + quote( "" );
279 final String [] tokens = create( input ).getTokens();
280
281 assertEquals( 1, tokens.length );
282 assertEquals( "xxxyyy", tokens[ 0 ] );
283 }
284
285 public void
286 testQuotedString()
287 throws TokenizerException
288 {
289 final String input = quote( "hello there" ) + " " + quote( "another" ) + " " +
290 quote( "3" ) + " " + quote( "words" ) + " ";
291 final String [] tokens = create( input ).getTokens();
292
293 assertEquals( 5, tokens.length );
294 assertEquals( "hello there", tokens[ 0 ] );
295 assertEquals( "another", tokens[ 1 ] );
296 assertEquals( "3", tokens[ 2 ] );
297 assertEquals( "words", tokens[ 3 ] );
298 assertEquals( "", tokens[ 4 ] );
299 }
300
301 public void
302 testQuotedDelim()
303 throws TokenizerException
304 {
305 final String input = quote( " " ) + " " + quote( " " ) + BACKSLASH + " ";
306 final String [] tokens = create( input ).getTokens();
307
308 assertEquals( 2, tokens.length );
309 assertEquals( " ", tokens[ 0 ] );
310 assertEquals( " ", tokens[ 1 ] );
311 }
312
313
314 public void
315 testTrailingEscapedDelim()
316 throws TokenizerException
317 {
318 final String input = "x." + BACKSLASH + ".";
319 final String [] tokens = new TokenizerImpl( input,
320 ".", false, BACKSLASH, BACKSLASH + "." ).getTokens();
321
322 assertEquals( 2, tokens.length );
323 assertEquals( "x", tokens[ 0 ] );
324 assertEquals( ".", tokens[ 1 ] );
325 }
326
327 public void
328 testLegalUnicodeSequence()
329 throws TokenizerException
330 {
331 final String input = BACKSLASH + "u0020";
333 final String [] tokens = create( input ).getTokens();
334 assertEquals( 1, tokens.length );
335 assertEquals( " ", tokens[ 0 ] );
336 }
337
338 public void
339 testIllegalUnicodeSequence()
340 throws TokenizerException
341 {
342 final String input1 = BACKSLASH + "u";
343 final String input2 = input1 + "zzzz";
344 final String input3 = input1 + "abcx";
345
346 try
347 {
348 create( input1 ).getTokens();
349 fail( "expected to fail: " + input1);
350 create( input2 ).getTokens();
351 fail( "expected to fail: " + input2);
352 create( input3 ).getTokens();
353 fail( "expected to fail: " + input3);
354 }
355 catch( TokenizerException e )
356 {
357 }
358 }
359
360
361 public void
362 testIllegalEscapeSequence()
363 throws TokenizerException
364 {
365 final String input = BACKSLASH + "x";
366 try
367 {
368 final String [] tokens = create( input ).getTokens();
369 fail( "expected to fail: " + input);
370 }
371 catch( TokenizerException e )
372 {
373 }
374 }
375
376 public void
377 testUnterminatedLiteralString()
378 throws TokenizerException
379 {
380 final String input1 = "" + QUOTE;
381 final String input2 = QUOTE + "xxx";
382 final String input3 = "xxx" + QUOTE;
383 try
384 {
385 create( input1 ).getTokens();
386 fail( "expected to fail: " + input1);
387
388 create( input2 ).getTokens();
389 fail( "expected to fail: " + input2);
390
391 create( input3 ).getTokens();
392 fail( "expected to fail: " + input3);
393 }
394 catch( TokenizerException e )
395 {
396 }
397 }
398
399
400
402
403
404 protected void
405 setUp()
406 {
407 }
408
409 protected void
410 tearDown()
411 {
412 }
413
414 };
415
416
417
418 |
Popular Tags |
Java API By Example, From Geeks To Geeks.