KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > netbeans > lib > lexer > test > simple > SimpleLexerBatchTest


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */

19
20 package org.netbeans.lib.lexer.test.simple;
21
22 import junit.framework.TestCase;
23 import org.netbeans.api.lexer.Language;
24 import org.netbeans.api.lexer.Token;
25 import org.netbeans.api.lexer.TokenHierarchy;
26 import org.netbeans.api.lexer.TokenId;
27 import org.netbeans.api.lexer.TokenSequence;
28 import org.netbeans.lib.lexer.test.LexerTestUtilities;
29
30 /**
31  * Test several simple lexer impls.
32  *
33  * @author mmetelka
34  */

35 public class SimpleLexerBatchTest extends TestCase {
36
37     public SimpleLexerBatchTest(String JavaDoc testName) {
38         super(testName);
39     }
40
41     protected void setUp() throws java.lang.Exception JavaDoc {
42     }
43
44     protected void tearDown() throws java.lang.Exception JavaDoc {
45     }
46
47     public void test() {
48         String JavaDoc commentText = "/* test comment */";
49         String JavaDoc text = "abc+ " + commentText + "def public publica publi static x";
50         int commentTextStartOffset = 5;
51         TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language());
52         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
53         assertTrue(ts.moveNext());
54         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0);
55         assertTrue(ts.moveNext());
56         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3);
57         assertTrue(ts.moveNext());
58         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", 4);
59         assertTrue(ts.moveNext());
60         int offset = commentTextStartOffset;
61         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, offset);
62         offset += commentText.length();
63         int commentIndex = ts.index();
64
65         assertTrue(ts.moveNext());
66         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset);
67         assertTrue(ts.moveNext());
68         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", -1);
69         assertTrue(ts.moveNext());
70         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PUBLIC, "public", -1);
71         assertTrue(ts.moveNext());
72         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", -1);
73         assertTrue(ts.moveNext());
74         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "publica", -1);
75         assertTrue(ts.moveNext());
76         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", -1);
77         assertTrue(ts.moveNext());
78         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "publi", -1);
79         assertTrue(ts.moveNext());
80         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", -1);
81         assertTrue(ts.moveNext());
82         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.STATIC, "static", -1);
83         assertTrue(ts.moveNext());
84         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", -1);
85         assertTrue(ts.moveNext());
86         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "x", -1);
87         assertFalse(ts.moveNext());
88
89         // Go back to block comment
90
assertEquals(0, ts.moveIndex(commentIndex));
91         assertTrue(ts.moveNext());
92         LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset);
93
94         // Test embedded token sequence
95
TokenSequence<? extends TokenId> embedded = ts.embedded();
96         assertNotNull("Null embedded sequence", embedded);
97         assertTrue(embedded.moveNext());
98         offset = commentTextStartOffset + 2; // skip "/*"
99
LexerTestUtilities.assertTokenEquals(embedded, SimplePlainTokenId.WHITESPACE, " ", offset);
100         offset += 1;
101         assertTrue(embedded.moveNext());
102         LexerTestUtilities.assertTokenEquals(embedded, SimplePlainTokenId.WORD, "test", offset);
103         offset += 4;
104         assertTrue(embedded.moveNext());
105         LexerTestUtilities.assertTokenEquals(embedded, SimplePlainTokenId.WHITESPACE, " ", offset);
106         offset += 1;
107         assertTrue(embedded.moveNext());
108         LexerTestUtilities.assertTokenEquals(embedded, SimplePlainTokenId.WORD, "comment", offset);
109         offset += 7;
110         assertTrue(embedded.moveNext());
111         LexerTestUtilities.assertTokenEquals(embedded, SimplePlainTokenId.WHITESPACE, " ", offset);
112         assertFalse(embedded.moveNext());
113
114     }
115     
116     public void testPerf() {
117         StringBuilder JavaDoc sb = new StringBuilder JavaDoc();
118         for (int i = 0; i < 7000; i++) {
119             sb.append("public static x + y /* test comment */ abc * def\n");
120         }
121         String JavaDoc text = sb.toString();
122
123         long tm;
124         Language<SimpleTokenId> language = SimpleTokenId.language();
125         tm = System.currentTimeMillis();
126         TokenHierarchy<?> hi = TokenHierarchy.create(text, language);
127         tm = System.currentTimeMillis() - tm;
128         assertTrue("Timeout tm = " + tm + "msec", tm < 100); // Should be fast
129

130         tm = System.currentTimeMillis();
131         TokenSequence<? extends TokenId> ts = hi.tokenSequence();
132         tm = System.currentTimeMillis() - tm;
133         assertTrue("Timeout tm = " + tm + "msec", tm < 100); // Should be fast
134

135         // Fetch 2 initial tokens - should be lexed lazily
136
tm = System.currentTimeMillis();
137         ts.moveNext();
138         ts.token();
139         ts.moveNext();
140         ts.token();
141         tm = System.currentTimeMillis() - tm;
142         assertTrue("Timeout tm = " + tm + "msec", tm < 100); // Should be fast
143

144         tm = System.currentTimeMillis();
145         ts.moveIndex(0);
146         int cntr = 1; // On the first token
147
while (ts.moveNext()) {
148             Token t = ts.token();
149             cntr++;
150         }
151         tm = System.currentTimeMillis() - tm;
152         assertTrue("Timeout tm = " + tm + "msec", tm < 1000); // Should be fast
153
System.out.println("Lexed input " + text.length()
154                 + " chars long and created " + cntr + " tokens in " + tm + " ms.");
155     }
156     
157 }
158
Popular Tags