KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > fri > patterns > interpreter > parsergenerator > Lexer


1 package fri.patterns.interpreter.parsergenerator;
2
3 import java.util.List JavaDoc;
4 import java.util.Map JavaDoc;
5 import java.io.PrintStream JavaDoc;
6 import java.io.IOException JavaDoc;
7
8 /**
9     A Lexer reads input bytes (InputStream) or characters (Reader), until
10     one of its terminals is fulfilled. This happens when the Parser calls
11     <i>getNextToken()</i>. The terminals will be set by the Parser on init.
12     <p>
13     Usage:
14     <pre>
15     SyntaxSeparation separation = new SyntaxSeparation(new Syntax(myRules));
16     LexerBuilder builder = new LexerBuilder(separation.getLexerSyntax(), separation.getIgnoredSymbols());
17     Lexer lexer = builder.getLexer(inputStream);
18     lexer.setTerminals(separation.getTokenSymbols());
19     Token token;
20     do {
21         token = lexer.getNextToken(null);
22         System.err.println("token.symbol="+token.symbol+", text >"+token.text+"<");
23     }
24     while (token.symbol != null && Token.isEpsilon(token) == false);
25     boolean error = token.symbol == null;
26     </pre>
27
28     @see fri.patterns.interpreter.parsergenerator.lexer.LexerImpl
29     @author (c) 2000, Fritz Ritzberger
30 */
31
32 public interface Lexer
33 {
34     /**
35         Set the input to be processed by the Lexer.
36         @param text can be String, StringBuffer, File, InputStream, Reader.
37     */
38     public void setInput(Object JavaDoc text) throws IOException JavaDoc;
39     
40     /**
41         Tells the Lexer the terminals (tokens) to scan, called on init. Every terminal is a String
42         that satisfies the facts defined in <i>Token.isTerminal()</i> (EPSILON or delimited by quotes).
43         @param terminals List of String containing all terminals of the parser syntax.
44     */
45     public void setTerminals(List JavaDoc terminals);
46
47     /**
48         Returns the next token from input. This is done trying to satisy the parser hints, or
49         by using contained character consumers, which are obtained by the lexer strategy.
50         @param tokenSymbols a Map that contains token symbols (in "key" field) expected by the Parser, can be null (slower).
51         @return a Token with a terminal symbol and its instance text, or a Token with null symbol for error.
52     */
53     public Token getNextToken(Map JavaDoc tokenSymbols) throws IOException JavaDoc;
54
55     /** Reset the Lexer for another pass. */
56     public void clear();
57
58
59
60
61     /**
62      * A way to receive every parsing syntax Token the Lexer reads, even it is ignored.
63      * The implementer can install itself by calling <i>addTokenListener()</i>.
64      */
65     public interface TokenListener
66     {
67         /**
68          * The implementer receives every token the lexer reads, even if it is ignored.
69          * @param token the Token that was successfully scanned from input.
70          * @param ignored true when this Token will be thrown away (not passed to Parser), else false.
71          */
72         public void tokenReceived(Token token, boolean ignored);
73     }
74     
75     /**
76      * Installs a TokenListener that wants to know about every read Token, even it is ignored.
77      * @param listener the Lexer.Listener implementation to install.
78      */
79     public void addTokenListener(Lexer.TokenListener listener);
80
81     /**
82      * Removes a TokenListener from this Lexer.
83      * @param listener the Lexer.Listener implementation to remove.
84      */
85     public void removeTokenListener(Lexer.TokenListener listener);
86
87
88
89
90     // debug methods
91
92     /** Dump the current text and the scan position. */
93     public void dump(PrintStream JavaDoc out);
94
95     /** Turn on and off debug mode. */
96     public void setDebug(boolean debug);
97
98 }
99
Popular Tags