KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > com > sun > enterprise > admin > util > TokenizerTest


1 /*
2  * The contents of this file are subject to the terms
3  * of the Common Development and Distribution License
4  * (the License). You may not use this file except in
5  * compliance with the License.
6  *
7  * You can obtain a copy of the license at
8  * https://glassfish.dev.java.net/public/CDDLv1.0.html or
9  * glassfish/bootstrap/legal/CDDLv1.0.txt.
10  * See the License for the specific language governing
11  * permissions and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL
14  * Header Notice in each file and include the License file
15  * at glassfish/bootstrap/legal/CDDLv1.0.txt.
16  * If applicable, add the following below the CDDL Header,
17  * with the fields enclosed by brackets [] replaced by
18  * you own identifying information:
19  * "Portions Copyrighted [year] [name of copyright owner]"
20  *
21  * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
22  */

23
24 /*
25  * Copyright 2004-2005 Sun Microsystems, Inc. All rights reserved.
26  * Use is subject to license terms.
27  */

28
29  
30 /*
31  * $Header: /cvs/glassfish/admin-core/util/tests/com/sun/enterprise/admin/util/TokenizerTest.java,v 1.2 2005/12/25 03:53:29 tcfujii Exp $
32  * $Revision: 1.2 $
33  * $Date: 2005/12/25 03:53:29 $
34  */

35  
36 package com.sun.enterprise.admin.util;
37
38 import java.util.Iterator JavaDoc;
39
40 import junit.framework.TestCase;
41
42
43
44 public class TokenizerTest extends TestCase
45 {
46         public
47     TokenizerTest( String JavaDoc name )
48     {
49         super( name );
50     }
51     
52     public final static char TAB = '\t';
53     public final static char BACKSLASH = '\\';
54     public final static char QUOTE_CHAR = '\"';
55     public final static String JavaDoc WHITE_SPACE = " " + TAB;
56     public final static char ESCAPE_CHAR = BACKSLASH;
57     public final static String JavaDoc ESCAPABLE_CHARS = "" + ESCAPE_CHAR + WHITE_SPACE + QUOTE_CHAR;
58     
59         Tokenizer
60     create( String JavaDoc input, boolean multipleDelimsCountAsOne )
61         throws TokenizerException
62     {
63         return( new TokenizerImpl( input,
64             WHITE_SPACE, multipleDelimsCountAsOne, ESCAPE_CHAR, ESCAPABLE_CHARS) );
65     }
66     
67         Tokenizer
68     create( String JavaDoc input )
69         throws TokenizerException
70     {
71         return( new TokenizerImpl( input,
72             WHITE_SPACE, false, ESCAPE_CHAR, ESCAPABLE_CHARS) );
73     }
74     
75         public void
76     testEmpty()
77         throws TokenizerException
78     {
79         final Tokenizer tk = create( "" );
80         
81         assertEquals( "expecting no tokens", 0, tk.getTokens().length );
82     }
83     
84     
85         public void
86     testLeadingDelim()
87         throws TokenizerException
88     {
89         final String JavaDoc input = WHITE_SPACE.charAt( 0 ) + "hello";
90         final String JavaDoc [] tokens = create( input, false ).getTokens();
91         
92         assertEquals( 2, tokens.length );
93         assertEquals( "", tokens[ 0 ] );
94         assertEquals( "hello", tokens[ 1 ] );
95     }
96     
97     
98         public void
99     testSingleDelimOnly()
100         throws TokenizerException
101     {
102         final String JavaDoc input = "" + WHITE_SPACE.charAt( 0 );
103         final String JavaDoc [] tokens = create( input, false ).getTokens();
104         
105         assertEquals( 2, tokens.length );
106         assertEquals( "", tokens[ 0 ] );
107         assertEquals( "", tokens[ 1 ] );
108     }
109     
110         public void
111     testOnlyDelims()
112         throws TokenizerException
113     {
114         final String JavaDoc input = WHITE_SPACE + WHITE_SPACE + WHITE_SPACE;
115         final String JavaDoc [] tokens = create( input, false ).getTokens();
116         
117         assertEquals( (WHITE_SPACE.length() * 3) + 1, tokens.length );
118     }
119     
120         public void
121     testSingleToken()
122         throws TokenizerException
123     {
124         final String JavaDoc input = "hello";
125         final String JavaDoc [] tokens = create( input, false ).getTokens();
126         
127         assertEquals( "expecting 1 token", 1, tokens.length );
128         assertEquals( "expecting " + input, input, tokens[ 0 ] );
129     }
130
131         public void
132     testMultipleTokens()
133         throws TokenizerException
134     {
135         final String JavaDoc input = "hello there 1 2 3 4 5";
136         final String JavaDoc[] tokens = create( input ).getTokens();
137         
138         assertEquals( "expecting 7 tokens", 7, tokens.length );
139         assertEquals( tokens[ 0 ], "hello" );
140         assertEquals( tokens[ 1 ], "there" );
141         assertEquals( tokens[ 2 ], "1" );
142         assertEquals( tokens[ 3 ], "2" );
143         assertEquals( tokens[ 4 ], "3" );
144         assertEquals( tokens[ 5 ], "4" );
145         assertEquals( tokens[ 6 ], "5" );
146     }
147     
148         public void
149     testWhiteSpaceEquality()
150         throws TokenizerException
151     {
152         final String JavaDoc input1 = "hello there 1 2 3 4 5";
153         final String JavaDoc input2 = "hello\tthere 1\t2 3 4\t5";
154         final String JavaDoc[] tokens1 = create( input1 ).getTokens();
155         final String JavaDoc[] tokens2 = create( input2 ).getTokens();
156         
157         
158         assertEquals( tokens1.length, tokens2.length );
159         for( int i = 0; i < tokens1.length; ++i )
160         {
161             assertEquals( tokens1[ i ], tokens2[ i ] );
162         }
163     }
164     
165         public void
166     testEscapedNewlineCR()
167         throws TokenizerException
168     {
169         final String JavaDoc TEST = "test" + BACKSLASH + "n" + BACKSLASH + "r" + BACKSLASH + "t";
170         final String JavaDoc [] tokens = create( TEST ).getTokens();
171         
172         assertEquals( 1, tokens.length );
173         assertEquals( "test\n\r\t", tokens[ 0 ] );
174     }
175     
176         public void
177     testEscaping()
178         throws TokenizerException
179     {
180         // create a String which each escapable character is represented
181
final StringBuffer JavaDoc b = new StringBuffer JavaDoc();
182         for( int i = 0; i < ESCAPABLE_CHARS.length(); ++i )
183         {
184             b.append( "\\" + ESCAPABLE_CHARS.charAt( i ) );
185         }
186         final String JavaDoc [] tokens = create( b.toString() ).getTokens();
187         
188         assertEquals( "expecting 1 token", 1, tokens.length );
189         assertEquals( "expecting match", ESCAPABLE_CHARS, tokens[ 0 ] );
190     }
191     
192     
193         public void
194     testTrailingDelimiter1()
195         throws TokenizerException
196     {
197         final String JavaDoc input = "hello" + WHITE_SPACE.charAt( 0 );
198         final String JavaDoc [] tokens = create( input, false ).getTokens();
199         
200         assertEquals( 2, tokens.length );
201     }
202     
203         public void
204     testTrailingDelimiter2()
205         throws TokenizerException
206     {
207         final String JavaDoc input = "hello" + WHITE_SPACE.charAt( 0 ) +
208                             "\"there\"" + WHITE_SPACE.charAt( 0 );
209         final String JavaDoc [] tokens = create( input, false ).getTokens();
210         
211         assertEquals( 3, tokens.length );
212     }
213     
214         public void
215     testMultipleDelimsWithNoData()
216         throws TokenizerException
217     {
218         final String JavaDoc input = "" + WHITE_SPACE.charAt( 0 );
219         final String JavaDoc [] tokens = create( input, false ).getTokens();
220         
221         assertEquals( input.length() + 1, tokens.length );
222     }
223     
224     
225         public void
226     testMultipleDelimsAsOne()
227         throws TokenizerException
228     {
229         final String JavaDoc input = "HELLO" + WHITE_SPACE + "THERE" + WHITE_SPACE + WHITE_SPACE;
230         final String JavaDoc [] tokens = create( input, true ).getTokens();
231         
232         assertEquals( 3, tokens.length );
233     }
234
235     static final char QUOTE = '\"';
236         static String JavaDoc
237     quote( String JavaDoc s )
238     {
239         return( QUOTE + s + QUOTE );
240     }
241     
242
243         public void
244     testEmptyQuotedString()
245         throws TokenizerException
246     {
247         final String JavaDoc input = quote( "" );
248         final String JavaDoc [] tokens = create( input ).getTokens();
249         
250         assertEquals( 1, tokens.length );
251         assertEquals( "", tokens[ 0 ] );
252     }
253         public void
254     testEmptyQuotedStrings()
255         throws TokenizerException
256     {
257         final String JavaDoc input = quote( "" ) + quote( "" ) + quote( "" ) + quote( "" );
258         final String JavaDoc [] tokens = create( input ).getTokens();
259         
260         assertEquals( 1, tokens.length );
261         assertEquals( "", tokens[ 0 ] );
262     }
263     
264         public void
265     testEmptyQuotedStringsSeparatedByDelim()
266         throws TokenizerException
267     {
268         final String JavaDoc input = quote( "" ) + WHITE_SPACE.charAt( 0 ) + quote( "" );
269         final String JavaDoc [] tokens = create( input ).getTokens();
270         
271         assertEquals( 2, tokens.length );
272     }
273     
274         public void
275     testAdjacentQuotedStrings()
276         throws TokenizerException
277     {
278         final String JavaDoc input = quote( "xxx" ) + quote( "yyy" ) + quote( "" );
279         final String JavaDoc [] tokens = create( input ).getTokens();
280         
281         assertEquals( 1, tokens.length );
282         assertEquals( "xxxyyy", tokens[ 0 ] );
283     }
284     
285         public void
286     testQuotedString()
287         throws TokenizerException
288     {
289         final String JavaDoc input = quote( "hello there" ) + " " + quote( "another" ) + " " +
290             quote( "3" ) + " " + quote( "words" ) + " ";
291         final String JavaDoc [] tokens = create( input ).getTokens();
292         
293         assertEquals( 5, tokens.length );
294         assertEquals( "hello there", tokens[ 0 ] );
295         assertEquals( "another", tokens[ 1 ] );
296         assertEquals( "3", tokens[ 2 ] );
297         assertEquals( "words", tokens[ 3 ] );
298         assertEquals( "", tokens[ 4 ] );
299     }
300     
301         public void
302     testQuotedDelim()
303         throws TokenizerException
304     {
305         final String JavaDoc input = quote( " " ) + " " + quote( " " ) + BACKSLASH + " ";
306         final String JavaDoc [] tokens = create( input ).getTokens();
307         
308         assertEquals( 2, tokens.length );
309         assertEquals( " ", tokens[ 0 ] );
310         assertEquals( " ", tokens[ 1 ] );
311     }
312     
313     
314         public void
315     testTrailingEscapedDelim()
316         throws TokenizerException
317     {
318         final String JavaDoc input = "x." + BACKSLASH + ".";
319         final String JavaDoc[] tokens = new TokenizerImpl( input,
320                                     ".", false, BACKSLASH, BACKSLASH + "." ).getTokens();
321                                     
322         assertEquals( 2, tokens.length );
323         assertEquals( "x", tokens[ 0 ] );
324         assertEquals( ".", tokens[ 1 ] );
325     }
326     
327         public void
328     testLegalUnicodeSequence()
329         throws TokenizerException
330     {
331         final String JavaDoc input = BACKSLASH + "u0020"; // unicode for the space char
332

333         final String JavaDoc[] tokens = create( input ).getTokens();
334         assertEquals( 1, tokens.length );
335         assertEquals( " ", tokens[ 0 ] );
336     }
337     
338         public void
339     testIllegalUnicodeSequence()
340         throws TokenizerException
341     {
342         final String JavaDoc input1 = BACKSLASH + "u";
343         final String JavaDoc input2 = input1 + "zzzz";
344         final String JavaDoc input3 = input1 + "abcx";
345         
346         try
347         {
348             create( input1 ).getTokens();
349             fail( "expected to fail: " + input1);
350             create( input2 ).getTokens();
351             fail( "expected to fail: " + input2);
352             create( input3 ).getTokens();
353             fail( "expected to fail: " + input3);
354         }
355         catch( TokenizerException e )
356         {
357         }
358     }
359     
360     
361         public void
362     testIllegalEscapeSequence()
363         throws TokenizerException
364     {
365         final String JavaDoc input = BACKSLASH + "x";
366         try
367         {
368             final String JavaDoc [] tokens = create( input ).getTokens();
369             fail( "expected to fail: " + input);
370         }
371         catch( TokenizerException e )
372         {
373         }
374     }
375
376         public void
377     testUnterminatedLiteralString()
378         throws TokenizerException
379     {
380         final String JavaDoc input1 = "" + QUOTE;
381         final String JavaDoc input2 = QUOTE + "xxx";
382         final String JavaDoc input3 = "xxx" + QUOTE;
383         try
384         {
385             create( input1 ).getTokens();
386             fail( "expected to fail: " + input1);
387             
388             create( input2 ).getTokens();
389             fail( "expected to fail: " + input2);
390             
391             create( input3 ).getTokens();
392             fail( "expected to fail: " + input3);
393         }
394         catch( TokenizerException e )
395         {
396         }
397     }
398
399
400     //-------------------------------------------------------------------------
401

402
403     
404         protected void
405     setUp()
406     {
407     }
408     
409         protected void
410     tearDown()
411     {
412     }
413     
414 };
415
416
417
418
Popular Tags