Java > Open Source Codes > org > netbeans > lib > java > lexer > JavaLexerBatchTest


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */
19
20 package org.netbeans.lib.java.lexer;
21
22 import junit.framework.TestCase;
23 import org.netbeans.api.java.lexer.JavaStringTokenId;
24 import org.netbeans.api.java.lexer.JavaTokenId;
25 import org.netbeans.api.java.lexer.JavadocTokenId;
26 import org.netbeans.api.lexer.PartType;
27 import org.netbeans.api.lexer.TokenHierarchy;
28 import org.netbeans.api.lexer.TokenId;
29 import org.netbeans.api.lexer.TokenSequence;
30 import org.netbeans.lib.lexer.test.LexerTestUtilities;
31
32 /**
33  * Test several simple lexer impls.
34  *
35  * @author mmetelka
36  */
37 public class JavaLexerBatchTest extends TestCase {
38
39     public JavaLexerBatchTest(String testName) {
40         super(testName);
41     }
42     
43     protected void setUp() throws java.lang.Exception {
44         // Set-up testing environment
45 LexerTestUtilities.setTesting(true);
46     }
47
48     protected void tearDown() throws java.lang.Exception {
49     }
50
51     public void testComments() {
52         String text = "/*ml-comment*//**//***//**\n*javadoc-comment*//* a";
53         TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
54         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
55         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BLOCK_COMMENT, "/*ml-comment*/");
56         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BLOCK_COMMENT, "/**/");
57         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.JAVADOC_COMMENT, "/***/");
58         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.JAVADOC_COMMENT, "/**\n*javadoc-comment*/");
59         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BLOCK_COMMENT, "/* a");
60         assertEquals(PartType.START, ts.token().partType());
61     }
62     
63     public void testIdentifiers() {
64         String text = "a ab aB2 2a x\nyZ\r\nz";
65         TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
66         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
67         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "a");
68         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
69         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "ab");
70         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
71         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "aB2");
72         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
73         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "2");
74         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "a");
75         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
76         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "x");
77         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, "\n");
78         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "yZ");
79         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, "\r\n");
80         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "z");
81     }
82     
83     public void testCharLiterals() {
84         String text = "'' 'a''' '\\'' '\\\\' '\\\\\\'' '\\n' 'a";
85         TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
86         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
87         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "''");
88         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
89         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'a'");
90         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "''");
91         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
92         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'\\''");
93         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
94         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'\\\\'");
95         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
96         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'\\\\\\''");
97         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
98         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'\\n'");
99         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
100         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR_LITERAL, "'a");
101         assertEquals(PartType.START, ts.token().partType());
102     }
103     
104     public void testStringLiterals() {
105         String text = "\"\" \"a\"\"\" \"\\\"\" \"\\\\\" \"\\\\\\\"\" \"\\n\" \"a";
106         TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
107         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
108         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"\"");
109         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
110         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"a\"");
111         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"\"");
112         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
113         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"\\\"\"");
114         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
115         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"\\\\\"");
116         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
117         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"\\\\\\\"\"");
118         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
119         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"\\n\"");
120         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
121         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"a");
122         assertEquals(PartType.START, ts.token().partType());
123     }
124     
125     public void testNumberLiterals() {
126         String text = "0 00 09 1 12 0L 1l 12L 0x1 0xf 0XdE 0Xbcy" +
127                 " 09.5 1.5f 2.5d 6d 7e3 6.1E-7f 0xa.5dp+12d .3";
128         TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
129         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
130         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "0");
131         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
132         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "00");
133         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
134         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "09");
135         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
136         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "1");
137         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
138         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "12");
139         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
140         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.LONG_LITERAL, "0L");
141         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
142         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.LONG_LITERAL, "1l");
143         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
144         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.LONG_LITERAL, "12L");
145         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
146         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "0x1");
147         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
148         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "0xf");
149         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
150         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "0XdE");
151         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
152         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "0Xbc");
153         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "y");
154         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
155         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.DOUBLE_LITERAL, "09.5");
156         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
157         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.FLOAT_LITERAL, "1.5f");
158         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
159         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.DOUBLE_LITERAL, "2.5d");
160         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
161         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.DOUBLE_LITERAL, "6d");
162         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
163         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.DOUBLE_LITERAL, "7e3");
164         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
165         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.FLOAT_LITERAL, "6.1E-7f");
166         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
167         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.DOUBLE_LITERAL, "0xa.5dp+12d");
168         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
169         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.DOUBLE_LITERAL, ".3");
170     }
171     
172     public void testOperators() {
173         String text = "^ ^= % %= * *= / /= = ==";
174         TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
175         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
176         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CARET, "^");
177         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
178         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CARETEQ, "^=");
179         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
180         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.PERCENT, "%");
181         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
182         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.PERCENTEQ, "%=");
183         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
184         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STAR, "*");
185         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
186         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STAREQ, "*=");
187         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
188         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.SLASH, "/");
189         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
190         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.SLASHEQ, "/=");
191         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
192         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.EQ, "=");
193         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
194         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.EQEQ, "==");
195     }
196
197     public void testKeywords() {
198         String text = "abstract assert boolean break byte case catch char class const continue " +
199             "default do double else enum extends final finally float for goto if " +
200             "implements import instanceof int interface long native new package " +
201             "private protected public return short static strictfp super switch " +
202             "synchronized this throw throws transient try void volatile while " +
203             "null true false";
204
205         TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
206         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
207         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.ABSTRACT, "abstract");
208         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
209         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.ASSERT, "assert");
210         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
211         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BOOLEAN, "boolean");
212         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
213         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BREAK, "break");
214         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
215         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.BYTE, "byte");
216         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
217         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CASE, "case");
218         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
219         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CATCH, "catch");
220         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
221         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CHAR, "char");
222         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
223         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CLASS, "class");
224         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
225         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CONST, "const");
226         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
227         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CONTINUE, "continue");
228         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
229         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.DEFAULT, "default");
230         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
231         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.DO, "do");
232         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
233         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.DOUBLE, "double");
234         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
235         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.ELSE, "else");
236         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
237         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.ENUM, "enum");
238         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
239         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.EXTENDS, "extends");
240         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
241         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.FINAL, "final");
242         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
243         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.FINALLY, "finally");
244         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
245         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.FLOAT, "float");
246         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
247         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.FOR, "for");
248         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
249         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.GOTO, "goto");
250         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
251         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IF, "if");
252         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
253         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IMPLEMENTS, "implements");
254         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
255         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IMPORT, "import");
256         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
257         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INSTANCEOF, "instanceof");
258         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
259         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT, "int");
260         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
261         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INTERFACE, "interface");
262         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
263         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.LONG, "long");
264         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
265         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.NATIVE, "native");
266         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
267         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.NEW, "new");
268         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
269         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.PACKAGE, "package");
270         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
271         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.PRIVATE, "private");
272         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
273         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.PROTECTED, "protected");
274         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
275         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.PUBLIC, "public");
276         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
277         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.RETURN, "return");
278         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
279         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.SHORT, "short");
280         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
281         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STATIC, "static");
282         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
283         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRICTFP, "strictfp");
284         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
285         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.SUPER, "super");
286         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
287         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.SWITCH, "switch");
288         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
289         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.SYNCHRONIZED, "synchronized");
290         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
291         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.THIS, "this");
292         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
293         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.THROW, "throw");
294         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
295         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.THROWS, "throws");
296         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
297         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.TRANSIENT, "transient");
298         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
299         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.TRY, "try");
300         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
301         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.VOID, "void");
302         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
303         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.VOLATILE, "volatile");
304         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
305         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHILE, "while");
306         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
307
308         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.NULL, "null");
309         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
310         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.TRUE, "true");
311         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
312         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.FALSE, "false");
313     }
314
315     public void testNonKeywords() {
316         String text = "abstracta assertx b br car dou doubl finall im i ifa inti throwsx";
317
318         TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
319         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
320         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "abstracta");
321         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
322         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "assertx");
323         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
324         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "b");
325         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
326         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "br");
327         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
328         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "car");
329         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
330         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "dou");
331         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
332         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "doubl");
333         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
334         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "finall");
335         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
336         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "im");
337         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
338         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "i");
339         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
340         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "ifa");
341         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
342         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "inti");
343         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
344         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "throwsx");
345     }
346     
347     public void testEmbedding() {
348         String text = "ddx \"d\\t\\br\" /** @see X */";
349         
350         TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
351         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
352         
353         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "ddx");
354         assertEquals(0, ts.offset());
355         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
356         assertEquals(3, ts.offset());
357         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\"d\\t\\br\"");
358         assertEquals(4, ts.offset());
359         
360         TokenSequence<? extends TokenId> es = ts.embedded();
361         
362         LexerTestUtilities.assertNextTokenEquals(es, JavaStringTokenId.TEXT, "d");
363         assertEquals(5, es.offset());
364         LexerTestUtilities.assertNextTokenEquals(es, JavaStringTokenId.TAB, "\\t");
365         assertEquals(6, es.offset());
366         LexerTestUtilities.assertNextTokenEquals(es, JavaStringTokenId.BACKSPACE, "\\b");
367         assertEquals(8, es.offset());
368         LexerTestUtilities.assertNextTokenEquals(es, JavaStringTokenId.TEXT, "r");
369         assertEquals(10, es.offset());
370         
371         assertFalse(es.moveNext());
372         
373         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
374         assertEquals(12, ts.offset());
375         LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.JAVADOC_COMMENT, "/** @see X */");
376         assertEquals(13, ts.offset());
377         
378         TokenSequence<? extends TokenId> ds = ts.embedded();
379         
380         LexerTestUtilities.assertNextTokenEquals(ds, JavadocTokenId.OTHER_TEXT, " ");
381         assertEquals(16, ds.offset());
382         LexerTestUtilities.assertNextTokenEquals(ds, JavadocTokenId.TAG, "@see");
383         assertEquals(17, ds.offset());
384         LexerTestUtilities.assertNextTokenEquals(ds, JavadocTokenId.OTHER_TEXT, " ");
385         assertEquals(21, ds.offset());
386         LexerTestUtilities.assertNextTokenEquals(ds, JavadocTokenId.IDENT, "X");
387         assertEquals(22, ds.offset());
388         LexerTestUtilities.assertNextTokenEquals(ds, JavadocTokenId.OTHER_TEXT, " ");
389         assertEquals(23, ds.offset());
390         
391         assertFalse(ds.moveNext());
392         
393         assertFalse(ts.moveNext());
394     }
395 }
396

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags