KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > fri > patterns > interpreter > parsergenerator > lexer > examples > ExampleLexer


1 package fri.patterns.interpreter.parsergenerator.lexer.examples;
2
3 import java.io.*;
4 import fri.patterns.interpreter.parsergenerator.Token;
5 import fri.patterns.interpreter.parsergenerator.Lexer;
6 import fri.patterns.interpreter.parsergenerator.lexer.*;
7 import fri.patterns.interpreter.parsergenerator.syntax.Syntax;
8 import fri.patterns.interpreter.parsergenerator.syntax.builder.SyntaxSeparation;
9
10 /**
11     Sample showing how to build a Lexer from StandardLexerRules building blocks.
12     This Lexer scans C/Java stringdefs, chardefs, integer, float, identifiers, comments,
13     and everything else as "verbatim".
14     
15     @author (c) 2002, Fritz Ritzberger
16 */
17
18 public abstract class ExampleLexer
19 {
20     /** Test main. */
21     public static void main(String JavaDoc [] args) {
22         String JavaDoc [][] rules = {
23             { "token", "stringdef" },
24             { "token", "chardef" },
25             { "token", "identifier" },
26             { "token", "float" },
27             { "token", "integer" },
28             { "token", "verbatim" },
29             { "verbatim", "char",
30                         Token.BUTNOT, "space",
31                         Token.BUTNOT, "newline",
32                         Token.BUTNOT, "letter",
33                         Token.BUTNOT, "digit",
34                         Token.BUTNOT, "stringdef",
35                         Token.BUTNOT, "chardef",
36                         Token.BUTNOT, "comment" },
37             { "ignored", "comment" },
38             { "ignored", "spaces" },
39             { "ignored", "newlines" },
40         };
41         Syntax syntax = new Syntax(rules);
42
43         try {
44             fri.util.TimeStopper ts = new fri.util.TimeStopper();
45
46             SyntaxSeparation separation = new SyntaxSeparation(syntax);
47             LexerBuilder builder = new LexerBuilder(separation.getLexerSyntax(), separation.getIgnoredSymbols());
48             Lexer lexer = builder.getLexer();
49             lexer.setDebug(true);
50             lexer.setTerminals(separation.getTokenSymbols());
51
52             System.err.println("time to build lexer was: "+ts.getInterval());
53
54             InputStream in = ExampleLexer.class.getResourceAsStream("ExampleLexer.java"); //Reader in = new FileReader(args[0]);
55 lexer.setInput(in);
56
57             System.err.println("time to build input was: "+ts.getInterval());
58             
59             Token t;
60             do {
61                 t = lexer.getNextToken(null);
62                 //System.err.println(ts.getInterval()+" was time to read token "+t.symbol+" -> "+t.text);
63
64                 if (t.symbol == null) {
65                     lexer.dump(System.err);
66                     throw new LexerException("Uninterpretable input!");
67                 }
68                 System.out.println(t.symbol+" "+">"+t.text+"<");
69             }
70             while (Token.isEpsilon(t) == false);
71
72             System.err.println("time to scan input was: "+ts.getTimeMillis());
73         }
74         catch (Exception JavaDoc e) {
75             e.printStackTrace();
76         }
77
78
79         // some complicated Java expressions, for test lexing this source file
80 char testChar1 = 'c';
81         char testChar2 = '\r';
82         char testChar3 = '\000';
83         char testChar4 = '\377';
84         String JavaDoc testString1 = "\\\"\\/* comment in string */";
85         String JavaDoc testString2 = "///*|<>@,;.:-_#'+*~´`ßäöü\"ÄÖÜ^°§%&/[][]{}";
86         float testFloat = 123.456E-12f;
87         double testDouble = .456d;
88         int testInt1 = 0X12aB;
89         int testInt2 = 07654;
90         System.err.println("Zahl 07654 ist: "+testInt2);
91         long testLong1 = 123456L;
92     }
93
94 }
95
Popular Tags