1 19 20 package org.netbeans.lib.lexer.test.simple; 21 22 import junit.framework.TestCase; 23 import org.netbeans.api.lexer.Language; 24 import org.netbeans.api.lexer.Token; 25 import org.netbeans.api.lexer.TokenHierarchy; 26 import org.netbeans.api.lexer.TokenId; 27 import org.netbeans.api.lexer.TokenSequence; 28 import org.netbeans.lib.lexer.test.LexerTestUtilities; 29 30 35 public class SimpleLexerBatchTest extends TestCase { 36 37 public SimpleLexerBatchTest(String testName) { 38 super(testName); 39 } 40 41 protected void setUp() throws java.lang.Exception { 42 } 43 44 protected void tearDown() throws java.lang.Exception { 45 } 46 47 public void test() { 48 String commentText = "/* test comment */"; 49 String text = "abc+ " + commentText + "def public publica publi static x"; 50 int commentTextStartOffset = 5; 51 TokenHierarchy<?> hi = TokenHierarchy.create(text, SimpleTokenId.language()); 52 TokenSequence<? extends TokenId> ts = hi.tokenSequence(); 53 assertTrue(ts.moveNext()); 54 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "abc", 0); 55 assertTrue(ts.moveNext()); 56 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PLUS, "+", 3); 57 assertTrue(ts.moveNext()); 58 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", 4); 59 assertTrue(ts.moveNext()); 60 int offset = commentTextStartOffset; 61 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, offset); 62 offset += commentText.length(); 63 int commentIndex = ts.index(); 64 65 assertTrue(ts.moveNext()); 66 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "def", offset); 67 assertTrue(ts.moveNext()); 68 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", -1); 69 assertTrue(ts.moveNext()); 70 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.PUBLIC, "public", -1); 71 assertTrue(ts.moveNext()); 72 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", -1); 73 assertTrue(ts.moveNext()); 74 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "publica", -1); 75 assertTrue(ts.moveNext()); 76 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", -1); 77 assertTrue(ts.moveNext()); 78 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "publi", -1); 79 assertTrue(ts.moveNext()); 80 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", -1); 81 assertTrue(ts.moveNext()); 82 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.STATIC, "static", -1); 83 assertTrue(ts.moveNext()); 84 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.WHITESPACE, " ", -1); 85 assertTrue(ts.moveNext()); 86 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.IDENTIFIER, "x", -1); 87 assertFalse(ts.moveNext()); 88 89 assertEquals(0, ts.moveIndex(commentIndex)); 91 assertTrue(ts.moveNext()); 92 LexerTestUtilities.assertTokenEquals(ts, SimpleTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset); 93 94 TokenSequence<? extends TokenId> embedded = ts.embedded(); 96 assertNotNull("Null embedded sequence", embedded); 97 assertTrue(embedded.moveNext()); 98 offset = commentTextStartOffset + 2; LexerTestUtilities.assertTokenEquals(embedded, SimplePlainTokenId.WHITESPACE, " ", offset); 100 offset += 1; 101 assertTrue(embedded.moveNext()); 102 LexerTestUtilities.assertTokenEquals(embedded, SimplePlainTokenId.WORD, "test", offset); 103 offset += 4; 104 assertTrue(embedded.moveNext()); 105 LexerTestUtilities.assertTokenEquals(embedded, SimplePlainTokenId.WHITESPACE, " ", offset); 106 offset += 1; 107 assertTrue(embedded.moveNext()); 108 LexerTestUtilities.assertTokenEquals(embedded, SimplePlainTokenId.WORD, "comment", offset); 109 offset += 7; 110 assertTrue(embedded.moveNext()); 111 LexerTestUtilities.assertTokenEquals(embedded, SimplePlainTokenId.WHITESPACE, " ", offset); 112 assertFalse(embedded.moveNext()); 113 114 } 115 116 public void testPerf() { 117 StringBuilder sb = new StringBuilder (); 118 for (int i = 0; i < 7000; i++) { 119 sb.append("public static x + y /* test comment */ abc * def\n"); 120 } 121 String text = sb.toString(); 122 123 long tm; 124 Language<SimpleTokenId> language = SimpleTokenId.language(); 125 tm = System.currentTimeMillis(); 126 TokenHierarchy<?> hi = TokenHierarchy.create(text, language); 127 tm = System.currentTimeMillis() - tm; 128 assertTrue("Timeout tm = " + tm + "msec", tm < 100); 130 tm = System.currentTimeMillis(); 131 TokenSequence<? extends TokenId> ts = hi.tokenSequence(); 132 tm = System.currentTimeMillis() - tm; 133 assertTrue("Timeout tm = " + tm + "msec", tm < 100); 135 tm = System.currentTimeMillis(); 137 ts.moveNext(); 138 ts.token(); 139 ts.moveNext(); 140 ts.token(); 141 tm = System.currentTimeMillis() - tm; 142 assertTrue("Timeout tm = " + tm + "msec", tm < 100); 144 tm = System.currentTimeMillis(); 145 ts.moveIndex(0); 146 int cntr = 1; while (ts.moveNext()) { 148 Token t = ts.token(); 149 cntr++; 150 } 151 tm = System.currentTimeMillis() - tm; 152 assertTrue("Timeout tm = " + tm + "msec", tm < 1000); System.out.println("Lexed input " + text.length() 154 + " chars long and created " + cntr + " tokens in " + tm + " ms."); 155 } 156 157 } 158 | Popular Tags |