Java > Open Source Codes > org > codehaus > groovy > syntax > lexer > LexerTest


1 package org.codehaus.groovy.syntax.lexer;
2
3 import groovy.util.GroovyTestCase;
4
5 import org.codehaus.groovy.syntax.Token;
6
7 public class LexerTest extends GroovyTestCase {
8
9     public void testNothing()
10     {
11     }
12
13 /*
14
15     private Lexer lexer;
16
17     public void testEndOfStream() throws Exception {
18         newLexer("");
19
20         assertEnd();
21
22         assertEnd();
23
24         assertEnd();
25
26         assertEnd();
27     }
28
29     public void testSingleLineComment_Newline() throws Exception {
30         newLexer("// I like cheese\ncheese");
31
32         assertNextToken(Token.NEWLINE, "<newline>");
33         assertNextToken(Token.IDENTIFIER, "cheese");
34
35         assertEnd();
36     }
37
38     public void testSingleLineComment_CarriageReturn() throws Exception {
39         newLexer("// I like cheese\rcheese");
40
41         assertNextToken(Token.NEWLINE, "<newline>");
42         assertNextToken(Token.IDENTIFIER, "cheese");
43
44         assertEnd();
45     }
46
47     public void testSingleLineComment_CarriageReturn_Newline() throws Exception {
48         newLexer("// I like cheese\r\ncheese");
49
50         assertNextToken(Token.NEWLINE, "<newline>");
51         assertNextToken(Token.IDENTIFIER, "cheese");
52
53         assertEnd();
54     }
55
56     public void testSingleLineHashComment_CarriageReturn() throws Exception {
57         newLexer("# I like cheese\rcheese");
58
59         assertNextToken(Token.NEWLINE, "<newline>");
60         assertNextToken(Token.IDENTIFIER, "cheese");
61
62         assertEnd();
63     }
64
65     public void testMultilineComment_MiddleOfLine() throws Exception {
66 */ // newLexer("cheese /* is */ toasty");
67 /*
68
69         assertNextToken(Token.IDENTIFIER, "cheese");
70
71         assertNextToken(Token.IDENTIFIER, "toasty");
72
73         assertEnd();
74     }
75
76     public void testMultilineComment_SpanningLines() throws Exception {
77 */ // newLexer("cheese /* is \n really */ toasty");
78 /*
79
80         assertNextToken(Token.IDENTIFIER, "cheese");
81
82         assertNextToken(Token.IDENTIFIER, "toasty");
83
84         assertEnd();
85     }
86
87     public void testMultilineComment_EmbeddedStarts() throws Exception {
88 */ // newLexer("cheese /* * * * / * / */ toasty");
89 /*
90
91         assertNextToken(Token.IDENTIFIER, "cheese");
92
93         assertNextToken(Token.IDENTIFIER, "toasty");
94
95         assertEnd();
96     }
97
98     public void testIgnoredWhitespace() throws Exception {
99         newLexer(" \r \n \r\n \n\r \t \t");
100
101         assertNextToken(Token.NEWLINE, "<newline>");
102         assertNextToken(Token.NEWLINE, "<newline>");
103         assertNextToken(Token.NEWLINE, "<newline>");
104
105         assertEnd();
106     }
107
108     public void testLeftCurlyBrace() throws Exception {
109         assertSimple("{", Token.LEFT_CURLY_BRACE);
110     }
111
112     public void testRightCurlyBrace() throws Exception {
113         assertSimple("}", Token.RIGHT_CURLY_BRACE);
114     }
115
116     public void testLeftSquareBracket() throws Exception {
117         assertSimple("[", Token.LEFT_SQUARE_BRACKET);
118     }
119
120     public void testRightSquareBracket() throws Exception {
121         assertSimple("]", Token.RIGHT_SQUARE_BRACKET);
122     }
123
124     public void testLeftParenthesis() throws Exception {
125         assertSimple("(", Token.LEFT_PARENTHESIS);
126     }
127
128     public void testRightParenthesis() throws Exception {
129         assertSimple(")", Token.RIGHT_PARENTHESIS);
130     }
131
132     public void testDot() throws Exception {
133         assertSimple(".", Token.DOT);
134     }
135
136     public void testDotDot() throws Exception {
137         assertSimple("..", Token.DOT_DOT);
138     }
139
140     public void testNot() throws Exception {
141         assertSimple("!", Token.NOT);
142     }
143
144     public void testCompareNotEqual() throws Exception {
145         assertSimple("!=", Token.COMPARE_NOT_EQUAL);
146     }
147
148     public void testEqual() throws Exception {
149         assertSimple("=", Token.EQUAL);
150     }
151
152     public void testCompareEqual() throws Exception {
153         assertSimple("==", Token.COMPARE_EQUAL);
154     }
155
156     public void testCompareIdentical() throws Exception {
157         assertSimple("===", Token.COMPARE_IDENTICAL);
158     }
159
160     public void testCompareLessThan() throws Exception {
161         assertSimple("<", Token.COMPARE_LESS_THAN);
162     }
163
164     public void testCompareLessThanEqual() throws Exception {
165         assertSimple("<=", Token.COMPARE_LESS_THAN_EQUAL);
166     }
167
168     public void testCompareGreaterThan() throws Exception {
169         assertSimple(">", Token.COMPARE_GREATER_THAN);
170     }
171
172     public void testCompareGreaterThanEqual() throws Exception {
173         assertSimple(">=", Token.COMPARE_GREATER_THAN_EQUAL);
174     }
175
176     public void testCompareTo() throws Exception {
177         assertSimple("<=>", Token.COMPARE_TO);
178     }
179
180     public void testNavigate() throws Exception {
181         assertSimple("->", Token.NAVIGATE);
182     }
183
184     public void testLogicalOr() throws Exception {
185         assertSimple("||", Token.LOGICAL_OR);
186     }
187
188     public void testPipe() throws Exception {
189         assertSimple("|", Token.PIPE);
190     }
191
192     public void testLogicalAnd() throws Exception {
193         assertSimple("&&", Token.LOGICAL_AND);
194     }
195
196     public void testAmpersand_UnexpectedCharacter() throws Exception {
197         newLexer("&a");
198
199         char[] expected = assertUnexpectedCharacter('a', 1, 2);
200
201         assertLength(1, expected);
202
203         assertContains('&', expected);
204     }
205
206     public void testPlus() throws Exception {
207         assertSimple("+", Token.PLUS);
208     }
209
210     public void testPlusPlus() throws Exception {
211         assertSimple("++", Token.PLUS_PLUS);
212     }
213
214     public void testPlusEqual() throws Exception {
215         assertSimple("+=", Token.PLUS_EQUAL);
216     }
217
218     public void testMinus() throws Exception {
219         assertSimple("-", Token.MINUS);
220     }
221
222     public void testMinusMinus() throws Exception {
223         assertSimple("--", Token.MINUS_MINUS);
224     }
225
226     public void testMinusEqual() throws Exception {
227         assertSimple("-=", Token.MINUS_EQUAL);
228     }
229
230     public void testDivide() throws Exception {
231         assertSimple("/", Token.DIVIDE);
232     }
233
234     public void testDivideEqual() throws Exception {
235         assertSimple("/=", Token.DIVIDE_EQUAL);
236     }
237
238     public void testMod() throws Exception {
239         assertSimple("%", Token.MOD);
240     }
241
242     public void testModEqual() throws Exception {
243         assertSimple("%=", Token.MOD_EQUAL);
244     }
245
246     public void testMultiply() throws Exception {
247         assertSimple("*", Token.MULTIPLY);
248     }
249
250     public void testMultiplyEqual() throws Exception {
251         assertSimple("*=", Token.MULTIPLY_EQUAL);
252     }
253
254     public void testColon() throws Exception {
255         assertSimple(":", Token.COLON);
256     }
257
258     public void testSemicolon() throws Exception {
259         assertSimple(";", Token.SEMICOLON);
260     }
261
262     public void testQuestion() throws Exception {
263         assertSimple("?", Token.QUESTION);
264     }
265
266     public void testDoubleQuoteString_Simple() throws Exception {
267         newLexer("\"cheese\"");
268
269         assertNextToken(Token.DOUBLE_QUOTE_STRING, "cheese");
270
271         assertEnd();
272     }
273
274     public void testDoubleQuoteString_EscapedDollar() throws Exception {
275         newLexer("\"foo $${bar}\"");
276
277         assertNextToken(Token.DOUBLE_QUOTE_STRING, "foo $${bar}");
278
279         assertEnd();
280     }
281
282     public void testDoubleQuoteString_EscapedTab() throws Exception {
283         newLexer("\"che\\tese\"");
284
285         assertNextToken(Token.DOUBLE_QUOTE_STRING, "che\tese");
286
287         assertEnd();
288     }
289
290     public void testDoubleQuoteString_EscapedNewline() throws Exception {
291         newLexer("\"che\\nese\"");
292
293         assertNextToken(Token.DOUBLE_QUOTE_STRING, "che\nese");
294
295         assertEnd();
296     }
297
298     public void testDoubleQuoteString_EscapedCarriageReturn() throws Exception {
299         newLexer("\"che\\rese\"");
300
301         assertNextToken(Token.DOUBLE_QUOTE_STRING, "che\rese");
302
303         assertEnd();
304     }
305
306     public void testDoubleQuoteString_EscapedOther() throws Exception {
307         newLexer("\"che\\bese\"");
308
309         assertNextToken(Token.DOUBLE_QUOTE_STRING, "chebese");
310
311         assertEnd();
312     }
313
314     public void testSingleQuoteString_Simple() throws Exception {
315         newLexer("'cheese'");
316
317         assertNextToken(Token.SINGLE_QUOTE_STRING, "cheese");
318
319         assertEnd();
320     }
321
322     public void testSingleQuoteString_EscapedTab() throws Exception {
323         newLexer("'che\\tese'");
324
325         assertNextToken(Token.SINGLE_QUOTE_STRING, "che\tese");
326
327         assertEnd();
328     }
329
330     public void testSingleQuoteString_EscapedNewline() throws Exception {
331         newLexer("'che\\nese'");
332
333         assertNextToken(Token.SINGLE_QUOTE_STRING, "che\nese");
334
335         assertEnd();
336     }
337
338     public void testSingleQuoteString_EscapedCarriageReturn() throws Exception {
339         newLexer("'che\\rese'");
340
341         assertNextToken(Token.SINGLE_QUOTE_STRING, "che\rese");
342
343         assertEnd();
344     }
345
346     public void testSingleQuoteString_EscapedOther() throws Exception {
347         newLexer("'che\\bese'");
348
349         assertNextToken(Token.SINGLE_QUOTE_STRING, "chebese");
350
351         assertEnd();
352     }
353
354     public void testUnterminatedStringLiteral_DoubleQuote_Newline() throws Exception {
355         newLexer("\"cheese\n");
356
357         try {
358             nextToken();
359             fail("should have thrown UnterminatedStringLiteralException");
360         }
361         catch (UnterminatedStringLiteralException e) {
362             // expected and correct
363         }
364     }
365
366     public void testUnterminatedStringLiteral_DoubleQuote_CarriageReturn() throws Exception {
367         newLexer("\"cheese\r");
368
369         try {
370             nextToken();
371             fail("should have thrown UnterminatedStringLiteralException");
372         }
373         catch (UnterminatedStringLiteralException e) {
374             // expected and correct
375         }
376     }
377
378     public void testUnterminatedStringLiteral_DoubleQuote_EndOfStream() throws Exception {
379         newLexer("\"cheese");
380
381         try {
382             nextToken();
383             fail("should have thrown UnterminatedStringLiteralException");
384         }
385         catch (UnterminatedStringLiteralException e) {
386             // expected and correct
387         }
388     }
389
390     public void testUnterminatedStringLiteral_SingleQuote_Newline() throws Exception {
391         newLexer("'cheese\n'");
392
393         try {
394             nextToken();
395             fail("should have thrown UnterminatedStringLiteralException");
396         }
397         catch (UnterminatedStringLiteralException e) {
398             // expected and correct
399         }
400     }
401
402     public void testUnterminatedStringLiteral_SingleQuote_CarriageReturn() throws Exception {
403         newLexer("'cheese\r'");
404
405         try {
406             nextToken();
407             fail("should have thrown UnterminatedStringLiteralException");
408         }
409         catch (UnterminatedStringLiteralException e) {
410             // expected and correct
411         }
412     }
413
414     public void testUnterminatedStringLiteral_SingleQuote_EndOfStream() throws Exception {
415         newLexer("'cheese");
416
417         try {
418             nextToken();
419             fail("should have thrown UnterminatedStringLiteralException");
420         }
421         catch (UnterminatedStringLiteralException e) {
422             // expected and correct
423         }
424     }
425
426     public void testIdentifier() throws Exception {
427         assertSimple("cheese", Token.IDENTIFIER);
428     }
429
430     public void testNumber_Integer() throws Exception {
431         assertSimple("42", Token.INTEGER_NUMBER);
432     }
433
434     public void testNumber_FloatingPoint() throws Exception {
435         assertSimple("42.84", Token.FLOAT_NUMBER);
436     }
437
438     public void testNumber_IntegerCall() throws Exception {
439         newLexer("42.cheese");
440
441         assertNextToken(Token.INTEGER_NUMBER, "42");
442         assertNextToken(Token.DOT, ".");
443         assertNextToken(Token.IDENTIFIER, "cheese");
444     }
445
446     public void testNumber_FloatCall() throws Exception {
447         newLexer("42.0.cheese");
448
449         assertNextToken(Token.FLOAT_NUMBER, "42.0");
450         assertNextToken(Token.DOT, ".");
451         assertNextToken(Token.IDENTIFIER, "cheese");
452     }
453
454 */ /*
455     public void testNumber_UnexpectedCharacter()
456         throws Exception
457     {
458         newLexer( "4.0cheese" );
459     
460         char[] expected = assertUnexpectedCharacter( 'c',
461                                                      1,
462                                                      4 );
463     
464         assertLength( 10,
465                       expected );
466     
467         assertContains( '0',
468                         expected );
469         assertContains( '1',
470                         expected );
471         assertContains( '2',
472                         expected );
473         assertContains( '3',
474                         expected );
475         assertContains( '4',
476                         expected );
477         assertContains( '5',
478                         expected );
479         assertContains( '6',
480                         expected );
481         assertContains( '7',
482                         expected );
483         assertContains( '8',
484                         expected );
485         assertContains( '9',
486                         expected );
487     }
488     */
489 /*
490
491     // ----------------------------------------------------------------------
492     // ----------------------------------------------------------------------
493
494     public void testKeyword_Abstract() throws Exception {
495         assertSimple("abstract", Token.KEYWORD_ABSTRACT);
496     }
497
498     public void testKeyword_As() throws Exception {
499         assertSimple("as", Token.KEYWORD_AS);
500     }
501
502     public void testKeyword_Break() throws Exception {
503         assertSimple("break", Token.KEYWORD_BREAK);
504     }
505
506     public void testKeyword_Case() throws Exception {
507         assertSimple("case", Token.KEYWORD_CASE);
508     }
509
510     public void testKeyword_Catch() throws Exception {
511         assertSimple("catch", Token.KEYWORD_CATCH);
512     }
513
514     public void testKeyword_Class() throws Exception {
515         assertSimple("class", Token.KEYWORD_CLASS);
516     }
517
518     public void testKeyword_Const() throws Exception {
519         assertSimple("const", Token.KEYWORD_CONST);
520     }
521
522     public void testKeyword_Continue() throws Exception {
523         assertSimple("continue", Token.KEYWORD_CONTINUE);
524     }
525
526     public void testKeyword_Default() throws Exception {
527         assertSimple("default", Token.KEYWORD_DEFAULT);
528     }
529
530     public void testKeyword_Do() throws Exception {
531         assertSimple("do", Token.KEYWORD_DO);
532     }
533
534     public void testKeyword_Else() throws Exception {
535         assertSimple("else", Token.KEYWORD_ELSE);
536     }
537
538     public void testKeyword_Extends() throws Exception {
539         assertSimple("extends", Token.KEYWORD_EXTENDS);
540     }
541
542     public void testKeyword_Final() throws Exception {
543         assertSimple("final", Token.KEYWORD_FINAL);
544     }
545
546     public void testKeyword_Finally() throws Exception {
547         assertSimple("finally", Token.KEYWORD_FINALLY);
548     }
549
550     public void testKeyword_For() throws Exception {
551         assertSimple("for", Token.KEYWORD_FOR);
552     }
553
554     public void testKeyword_Goto() throws Exception {
555         assertSimple("goto", Token.KEYWORD_GOTO);
556     }
557
558     public void testKeyword_If() throws Exception {
559         assertSimple("if", Token.KEYWORD_IF);
560     }
561
562     public void testKeyword_Implements() throws Exception {
563         assertSimple("implements", Token.KEYWORD_IMPLEMENTS);
564     }
565
566     public void testKeyword_Import() throws Exception {
567         assertSimple("import", Token.KEYWORD_IMPORT);
568     }
569
570     public void testKeyword_Instanceof() throws Exception {
571         assertSimple("instanceof", Token.KEYWORD_INSTANCEOF);
572     }
573
574     public void testKeyword_Interface() throws Exception {
575         assertSimple("interface", Token.KEYWORD_INTERFACE);
576     }
577
578     public void testKeyword_Native() throws Exception {
579         assertSimple("native", Token.KEYWORD_NATIVE);
580     }
581
582     public void testKeyword_New() throws Exception {
583         assertSimple("new", Token.KEYWORD_NEW);
584     }
585
586     public void testKeyword_Package() throws Exception {
587         assertSimple("package", Token.KEYWORD_PACKAGE);
588     }
589
590     public void testKeyword_Private() throws Exception {
591         assertSimple("private", Token.KEYWORD_PRIVATE);
592     }
593
594     public void testKeyword_Property() throws Exception {
595         assertSimple("property", Token.KEYWORD_PROPERTY);
596     }
597
598     public void testKeyword_Protected() throws Exception {
599         assertSimple("protected", Token.KEYWORD_PROTECTED);
600     }
601
602     public void testKeyword_Public() throws Exception {
603         assertSimple("public", Token.KEYWORD_PUBLIC);
604     }
605
606     public void testKeyword_Return() throws Exception {
607         assertSimple("return", Token.KEYWORD_RETURN);
608     }
609
610     public void testKeyword_Static() throws Exception {
611         assertSimple("static", Token.KEYWORD_STATIC);
612     }
613
614     public void testKeyword_Super() throws Exception {
615         assertSimple("super", Token.KEYWORD_SUPER);
616     }
617
618     public void testKeyword_Switch() throws Exception {
619         assertSimple("switch", Token.KEYWORD_SWITCH);
620     }
621
622     public void testKeyword_Synchronized() throws Exception {
623         assertSimple("synchronized", Token.KEYWORD_SYNCHRONIZED);
624     }
625
626     public void testKeyword_This() throws Exception {
627         assertSimple("this", Token.KEYWORD_THIS);
628     }
629
630     public void testKeyword_Throw() throws Exception {
631         assertSimple("throw", Token.KEYWORD_THROW);
632     }
633
634     public void testKeyword_Throws() throws Exception {
635         assertSimple("throws", Token.KEYWORD_THROWS);
636     }
637
638     public void testKeyword_Try() throws Exception {
639         assertSimple("try", Token.KEYWORD_TRY);
640     }
641
642     public void testKeyword_While() throws Exception {
643         assertSimple("while", Token.KEYWORD_WHILE);
644     }
645
646     public void testUnexpecteCharacterException() throws Exception {
647         newLexer("±");
648
649         try {
650             nextToken();
651             fail("should have thrown UnexpectedCharacterException");
652         }
653         catch (UnexpectedCharacterException e) {
654             // expected and correct
655             assertEquals('±', e.getCharacter());
656         }
657     }
658
659     // ----------------------------------------------------------------------
660     // ----------------------------------------------------------------------
661
662     protected void assertSimple(String text, int type) throws Exception {
663         newLexer(text);
664
665         assertNextToken(type, text);
666
667         assertEnd();
668     }
669
670     protected void assertNextToken(int type, String text) throws Exception {
671         Token token = this.lexer.nextToken();
672
673         assertNotNull(token);
674
675         assertEquals(type, token.getType());
676
677         //System.out.println("Expected: " + text + " but got: " + token.getText());
678         
679         assertEquals(text, token.getText());
680     }
681
682     protected void nextToken() throws Exception {
683         this.lexer.nextToken();
684     }
685
686     protected char[] assertUnexpectedCharacter(char c, int line, int column) throws Exception {
687         try {
688             this.lexer.nextToken();
689             fail("should have thrown UnexpectedCharacterException");
690         }
691         catch (UnexpectedCharacterException e) {
692             // expected and correct
693             assertEquals(c, e.getCharacter());
694
695             assertEquals(line, e.getLine());
696
697             assertEquals(column, e.getStartColumn());
698
699             return e.getExpected();
700         }
701
702         return new char[] {
703         };
704     }
705
706     protected void assertEnd() throws Exception {
707         assertNull(this.lexer.nextToken());
708     }
709
710     protected void newLexer(String text) {
711         StringCharStream in = new StringCharStream(text);
712
713         this.lexer = new Lexer(in);
714     }
715 */
716 }
717

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags