TokenSequenceTest


1   /*
2    * The contents of this file are subject to the terms of the Common Development
3    * and Distribution License (the License). You may not use this file except in
4    * compliance with the License.
5    *
6    * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7    * or http://www.netbeans.org/cddl.txt.
8    *
9    * When distributing Covered Code, include this CDDL Header Notice in each file
10   * and include the License file at http://www.netbeans.org/cddl.txt.
11   * If applicable, add the following below the CDDL Header, with the fields
12   * enclosed by brackets [] replaced by your own identifying information:
13   * "Portions Copyrighted [year] [name of copyright owner]"
14   *
15   * The Original Software is NetBeans. The Initial Developer of the Original
16   * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17   * Microsystems, Inc. All Rights Reserved.
18   */
19  
20  package org.netbeans.api.lexer;
21  
22  import java.io.IOException  ;
23  import java.io.Reader  ;
24  import java.io.StringReader  ;
25  import java.util.Collections  ;
26  import java.util.EnumSet  ;
27  import java.util.Set  ;
28  import org.netbeans.junit.NbTestCase;
29  import org.netbeans.lib.lexer.TokenList;
30  import org.netbeans.lib.lexer.test.LexerTestUtilities;
31  import org.netbeans.lib.lexer.test.simple.*;
32  import org.netbeans.lib.lexer.token.DefaultToken;
33  import org.netbeans.lib.lexer.token.TextToken;
34  
35  /**
36   * Test methods of token sequence.
37   *
38   * @author mmetelka
39   */
40  public class TokenSequenceTest extends NbTestCase {
41      
42      public TokenSequenceTest(String   testName) {
43          super(testName);
44      }
45      
46      protected void setUp() throws java.lang.Exception   {
47      }
48  
49      protected void tearDown() throws java.lang.Exception   {
50      }
51      
52      public void testMove() {
53          String   text = "abc+defg";
54          TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
55          TokenSequence<? extends TokenId> ts = hi.tokenSequence();
56          
57          assertNull(ts.token());
58  
59          assertTrue(ts.moveNext());
60          LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
61  
62          // Check movePrevious()
63          assertFalse(ts.movePrevious()); // cannot go below first token
64          
65          assertTrue(ts.moveNext()); // on "+"
66          assertTrue(ts.moveNext()); // on "defg"
67          assertFalse(ts.moveNext());
68          
69          assertEquals(ts.tokenCount(), 3);
70          
71          assertEquals(0, ts.move(0));
72          assertNull(ts.token());
73          assertTrue(ts.moveNext());
74          LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
75          assertEquals(2, ts.move(2));
76          assertNull(ts.token());
77          assertTrue(ts.moveNext());
78          LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
79          assertEquals(-1, ts.move(-1));
80          assertNull(ts.token());
81          assertTrue(ts.moveNext());
82          LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
83          assertEquals(0, ts.move(3));
84          assertNull(ts.token());
85          assertTrue(ts.moveNext());
86          LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
87          assertEquals(0, ts.move(4));
88          assertNull(ts.token());
89          assertTrue(ts.moveNext());
90          LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
91          assertEquals(1, ts.move(5));
92          assertNull(ts.token());
93          assertTrue(ts.moveNext());
94          LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
95          assertEquals(1, ts.move(9));
96          assertNull(ts.token());
97          assertFalse(ts.moveNext());
98          assertTrue(ts.movePrevious());
99          LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
100         assertEquals(92, ts.move(100));
101         assertNull(ts.token());
102         assertFalse(ts.moveNext());
103         assertTrue(ts.movePrevious());
104         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
105 
106         
107         // Test subsequences
108         TokenSequence<? extends TokenId> sub = ts.subSequence(1, 6);
109         assertTrue(sub.moveNext());
110         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 0);
111         assertTrue(sub.moveNext());
112         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.PLUS, "+", 3);
113         assertTrue(sub.moveNext());
114         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 4);
115         
116         sub = ts.subSequence(1, 6);
117         assertEquals(2, sub.move(6));
118 
119         sub = ts.subSequence(3);
120         assertTrue(sub.moveNext());
121         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.PLUS, "+", 3);
122         assertTrue(sub.moveNext());
123         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 4);
124 
125         sub = ts.subSequence(3, 3);
126         assertFalse(sub.moveNext());
127     }
128     
129     public void testMoveNextPrevious() {
130         String   text = "abc+defg-";
131         TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
132         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
133         assertEquals(0, ts.move(4));
134         assertNull(ts.token());
135         //assertEquals(4, ts.offset());
136         assertTrue(ts.movePrevious());
137         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
138         
139         ts = hi.tokenSequence();
140         ts.move(5);
141         assertTrue(ts.moveNext());
142         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
143         assertTrue(ts.movePrevious());
144         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
145 
146         // Move single char before token's end
147         ts = hi.tokenSequence();
148         ts.move(7);
149         assertTrue(ts.moveNext());
150         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 4);
151         assertTrue(ts.movePrevious());
152         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
153     
154         // Move past all tokens
155         ts = hi.tokenSequence();
156         ts.move(text.length());
157         assertTrue(ts.movePrevious());
158         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.MINUS, "-", text.length() - 1);
159 
160         // Move at the begining
161         ts = hi.tokenSequence();
162         ts.move(0);
163         assertFalse(ts.movePrevious());
164         assertTrue(ts.moveNext());
165         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
166     }
167     
168     public void testMoveIndex() {
169         String   text = "abc+defg-";
170         TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
171         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
172         assertEquals(0, ts.index());
173         assertNull(ts.token());
174         //assertEquals(4, ts.offset());
175         assertTrue(ts.moveNext());
176         assertEquals(0, ts.offset());
177         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
178         assertTrue(ts.moveNext());
179         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
180         assertEquals(3, ts.offset());
181         ts.move(2);
182         assertEquals(0, ts.index());
183         assertTrue(ts.moveNext());
184         assertEquals(0, ts.offset());
185         assertEquals(0, ts.index());
186         ts.moveIndex(1);
187         assertEquals(1, ts.index());
188         assertTrue(ts.moveNext());
189         assertEquals(1, ts.index());
190         assertEquals(3, ts.offset());
191     }
192 
193     public void testMoveSkipTokens() {
194         String   text = "-abc+defg--hi";
195         Set  <SimpleTokenId> skipTokenIds = 
196                 EnumSet.of(SimpleTokenId.PLUS, SimpleTokenId.MINUS);
197         
198         assertTrue(skipTokenIds.contains(SimpleTokenId.PLUS));
199         assertTrue(skipTokenIds.contains(SimpleTokenId.MINUS));
200         assertFalse(skipTokenIds.contains(SimpleTokenId.IDENTIFIER));
201 
202         TokenHierarchy<?> hi = TokenHierarchy.create(text, false,
203                 SimpleTokenId.language(), skipTokenIds, null);
204         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
205         
206         // Fail if no "move*" method called yet
207         assertNull(ts.token());
208 
209         assertTrue(ts.moveNext());
210         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
211         assertTrue(ts.moveNext());
212         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5);
213         assertTrue(ts.moveNext());
214         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
215         assertFalse(ts.moveNext());
216         assertEquals(3, ts.tokenCount());
217         assertEquals(0, ts.moveIndex(0));
218         assertTrue(ts.moveNext());
219         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
220         assertTrue(ts.moveNext());
221         assertEquals(0, ts.moveIndex(ts.tokenCount()));
222         assertTrue(ts.movePrevious());
223         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
224         assertFalse(ts.moveNext());
225 
226         // Check movePrevious()
227         assertTrue(ts.movePrevious()); // over "defg"
228         assertTrue(ts.movePrevious()); // over "abc"
229         assertFalse(ts.movePrevious()); // cannot go below first token
230         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
231         
232         assertEquals(-1, ts.move(0)); // below filtered-out token
233         assertNull(ts.token());
234         assertTrue(ts.moveNext());
235         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
236         assertEquals(0, ts.move(1));
237         assertNull(ts.token());
238         assertTrue(ts.moveNext());
239         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
240         assertEquals(1, ts.move(2));
241         assertNull(ts.token());
242         assertTrue(ts.moveNext());
243         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
244         assertEquals(3, ts.move(4));
245         assertNull(ts.token());
246         assertTrue(ts.moveNext());
247         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 1);
248         assertEquals(0, ts.move(5));
249         assertNull(ts.token());
250         assertTrue(ts.moveNext());
251         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "defg", 5);
252         assertEquals(1, ts.move(12));
253         assertNull(ts.token());
254         assertTrue(ts.moveNext());
255         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "hi", 11);
256         
257         
258         // Test subsequences
259         TokenSequence<? extends TokenId> sub = ts.subSequence(1, 6);
260         assertTrue(sub.moveNext());
261         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1);
262         assertTrue(sub.moveNext());
263         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
264         assertFalse(sub.moveNext());
265         
266         // Test moves to first and last tokens on subsequence
267         sub = ts.subSequence(1, 6);
268         assertEquals(0, sub.moveIndex(sub.tokenCount()));
269         assertEquals(2, sub.tokenCount()); // "abc" and "defg" (others filtered out
270         assertTrue(sub.movePrevious());
271         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
272         assertEquals(0, sub.moveIndex(0));
273         assertTrue(sub.moveNext());
274         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "abc", 1);
275         
276         sub = ts.subSequence(4);
277         assertTrue(sub.moveNext());
278         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "defg", 5);
279         assertTrue(sub.moveNext());
280         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
281         assertFalse(sub.moveNext());
282 
283         sub = ts.subSequence(12, 15);
284         assertTrue(sub.moveNext());
285         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
286 
287         sub = ts.subSequence(12, 15);
288         assertEquals(-2, sub.move(9));
289         assertNull(sub.token());
290         assertTrue(sub.moveNext());
291         LexerTestUtilities.assertTokenEquals(sub, SimpleTokenId.IDENTIFIER, "hi", 11);
292 
293         sub = ts.subSequence(13, 15);
294         assertFalse(sub.moveNext());
295         assertEquals(0, sub.moveIndex(0));
296         assertNull(sub.token());
297         assertFalse(sub.moveNext());
298         assertEquals(0, sub.moveIndex(sub.tokenCount()));
299         assertNull(sub.token());
300         assertEquals(0, sub.tokenCount());
301 
302         sub = ts.subSequence(-3, 1);
303         assertFalse(sub.moveNext());
304 
305         sub = ts.subSequence(13, 15);
306         assertEquals(5, sub.move(5));
307 
308         sub = ts.subSequence(-3, 1);
309         assertEquals(1, sub.move(1));
310 
311         // Reversed bounds => should be empty token sequence
312         sub = ts.subSequence(6, 1);
313         assertFalse(sub.moveNext());
314         sub = ts.subSequence(6, 1);
315         assertEquals(6, sub.move(6));
316     }
317 
318     public void DtestMoveSkipTokens2() throws IOException   {
319         String   text = "abc+defg--hi";
320         Set  <SimpleTokenId> skipTokenIds =
321                 EnumSet.of(SimpleTokenId.IDENTIFIER);
322         
323         assertFalse(skipTokenIds.contains(SimpleTokenId.PLUS));
324         assertFalse(skipTokenIds.contains(SimpleTokenId.MINUS));
325         assertTrue(skipTokenIds.contains(SimpleTokenId.IDENTIFIER));
326         
327         Reader   r = new StringReader  (text);
328         
329         try {
330             TokenHierarchy<?> hi = TokenHierarchy.create(r, SimpleTokenId.language(), skipTokenIds, null);
331             TokenSequence<? extends TokenId> ts = hi.tokenSequence();
332             
333             ts.tokenCount();
334         } finally {
335             r.close();
336         }
337     }
338     
339     public void testMoveEmpty() {
340         String   text = "";
341         TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
342         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
343 
344         // Expect no tokens
345         assertFalse(ts.moveNext());
346         assertNull(ts.token());
347         
348         assertEquals(0, ts.move(0));
349         assertEquals(10, ts.move(10));
350         
351         // Test subsequences
352         TokenSequence<? extends TokenId> sub = ts.subSequence(1, 6);
353         assertFalse(sub.moveNext());
354         sub = ts.subSequence(1, 6);
355         assertEquals(1, sub.move(1));
356         
357         sub = ts.subSequence(0);
358         assertFalse(sub.moveNext());
359         sub = ts.subSequence(0);
360         assertEquals(0, sub.move(0));
361 
362         sub = ts.subSequence(1);
363         assertFalse(sub.moveNext());
364         sub = ts.subSequence(1);
365         assertEquals(0, sub.move(0));
366     }
367     
368     public void testTokenSize() {
369         String   text = "abc+";
370         TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
371         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
372         
373         assertTrue(ts.moveNext());
374         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
375         assertTrue(ts.moveNext());
376         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
377         assertFalse(ts.moveNext());
378         
379         TokenList tokenList = LexerTestUtilities.tokenList(ts);
380         ts.moveIndex(0); // move before "abc"
381         assertTrue(ts.moveNext());
382         // Test DefaultToken size
383         assertSame(DefaultToken.class, ts.token().getClass());
384         assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,
385                 new Object  [] { tokenList, SimpleTokenId.IDENTIFIER});
386         // Test TextToken size
387         assertTrue(ts.moveNext());
388         assertSame(TextToken.class, ts.token().getClass());
389         assertSize("Token instance too big", Collections.singletonList(ts.token()), 24,
390                 new Object  [] { tokenList, SimpleTokenId.PLUS, "+"});
391     }
392 
393 }
394
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags