Lexer


1   // Copyright (c) 1999, 2004  Per M.A. Bothner.
2   // This is free software;  for terms and warranty disclaimer see ./COPYING.
3   
4   package gnu.text;
5   import java.io.*;
6   
7   /**
8    * Framework for implementing lexical scanners and parsers.
9    * @author  Per Bothner
10   */
11  
12  public class Lexer extends Reader
13  {
14    protected LineBufferedReader port;
15    protected boolean interactive;
16  
17    public Lexer(LineBufferedReader port)
18    {
19      this.port = port;
20    }
21  
22    public Lexer(LineBufferedReader port, SourceMessages messages)
23    {
24      this.port = port;
25      this.messages = messages;
26    }
27  
28    /** Enter a nested expression.
29     * This is used in interactive mode to control whether to continue
30     * past end of line, depending on whether the expression is incomplete.
31     * @param promptChar Used in prompt string to indicate type of nesting.
32     * @return The previous value of promptChar, to be passed to popNesting.
33     */
34    public char pushNesting (char promptChar)
35    {
36      nesting++;
37      LineBufferedReader port = getPort();
38      char save = port.readState;
39      port.readState = promptChar;
40      return save;
41    }
42  
43    /** Exit a nested expression, reversing pushNesting
44     * @param save Saved values return by prior pushNesting
45     */
46    public void popNesting (char save)
47    {
48      LineBufferedReader port = getPort();
49      port.readState = save;
50      nesting--;
51    }
52  
53    protected int nesting;
54  
55    public final LineBufferedReader getPort() { return port; }
56  
57    public void close() throws java.io.IOException  
58    {
59      port.close();
60    }
61  
62    public int read() throws java.io.IOException  
63    {
64      return port.read();
65    }
66  
67    public int read(char[] buf, int offset, int length)
68      throws java.io.IOException  
69    {
70      return port.read(buf, offset, length);
71    }
72  
73    public void unread(int ch) throws java.io.IOException  
74    {
75      if (ch >= 0)
76        port.unread();
77    }
78  
79    public int peek() throws java.io.IOException  
80    {
81      return port.peek();
82    }
83  
84    public void skip() throws java.io.IOException  
85    {
86      port.skip();
87    }
88  
89    protected void unread() throws java.io.IOException  
90    {
91      port.unread();
92    }
93  
94    protected void unread_quick() throws java.io.IOException  
95    {
96      port.unread_quick();
97    }
98  
99    /**
100    * Check if the next character matches a given character.
101    * @param ch The character to match against.
102    * @return if the character read matches
103    * On a match, the position is advanced following that character.
104    */
105   public boolean checkNext(char ch)
106       throws java.io.IOException  
107   {
108     int r = port.read();
109     if (r == ch)
110     return true;
111     if (r >= 0)
112       port.unread_quick();
113     return false;
114   }
115 
116   protected void skip_quick() throws java.io.IOException  
117   {
118     port.skip_quick();
119   }
120 
121   SourceMessages messages = null;
122 
123   public SourceMessages getMessages () { return messages; }
124   public void setMessages (SourceMessages messages)
125   { this.messages = messages; }
126 
127   /** Returns true if any error were seen.  Prints and clears the errors.
128    * @param out where to write the error message to
129    * @param max maximum number of messages to print (can be 0) */
130   public boolean checkErrors(PrintWriter out, int max)
131   {
132     return messages != null && messages.checkErrors(out, max);
133   }
134 
135   public SourceError getErrors()
136   { return messages == null ? null : messages.getErrors(); }
137 
138   public boolean seenErrors()
139   { return messages != null && messages.seenErrors(); }
140 
141   public void clearErrors() { if (messages != null) messages.clearErrors(); }
142 
143   public void error(char severity, String   filename, int line, int column,
144             String   message)
145   {
146     if (messages == null)
147       messages = new SourceMessages();
148     messages.error(severity, filename, line, column, message);
149   }
150 
151   public void error(char severity, String   message)
152   {
153     int line = port.getLineNumber();
154     int column = port.getColumnNumber();
155     error(severity, port.getName(), line + 1, column >= 0 ? column + 1 : 0,
156       message);
157   }
158 
159   public void error(String   message)
160   {
161     error('e', message);
162   }
163 
164   public void fatal(String   message) throws SyntaxException
165   {
166     error('f', message);
167     throw new SyntaxException(messages);
168   }
169 
170   public void eofError(String   msg) throws SyntaxException
171   {
172     fatal(msg);
173   }
174 
175   public void eofError(String   message, int startLine, int startColumn)
176     throws SyntaxException
177   {
178     error('f', port.getName(), startLine, startColumn, message);
179     throw new SyntaxException(messages);
180   }
181 
182   /** Read an optional signed integer.
183    * If there is no integer in the input stream, return 1.
184    * For excessively large exponents, return Integer.MIN_VALUE
185    * or Integer.MAX_VALUE.
186    */
187   public int readOptionalExponent()
188        throws java.io.IOException  
189   {
190     int sign = read();
191     boolean overflow = false;
192     int c;
193     if (sign == '+' || sign == '-')
194       c = read();
195     else
196       {
197     c = sign;
198     sign = 0;
199       }
200     int value;
201     if (c < 0 || (value = Character.digit ((char)c, 10)) < 0)
202       {
203     if (sign != 0)
204       error("exponent sign not followed by digit");
205     value = 1;
206       }
207     else
208       {
209     int max = (Integer.MAX_VALUE - 9) / 10;
210     for (;;)
211       {
212         c = read();
213         int d = Character.digit ((char)c, 10);
214         if (d < 0)
215           break;
216         if (value > max)
217           overflow = true;
218         value = 10 * value + d;
219       }
220       }
221     if (c >= 0)
222       unread(c);
223     if (sign == '-')
224       value = -value;
225     if (overflow)
226       return sign == '-' ? Integer.MIN_VALUE : Integer.MAX_VALUE;
227     return value;
228   }
229 
230   /** Read digits, up to the first non-digit or the buffer limit
231     * @return the digits seen as a non-negative long, or -1 on overflow
232     */
233   public static long readDigitsInBuffer (LineBufferedReader port, int radix)
234   {
235     long ival = 0;
236     boolean overflow = false;
237     long max_val = Long.MAX_VALUE / radix;
238     int i = port.pos;
239     if (i >= port.limit)
240       return 0;
241     for (;;)
242       {
243     char c = port.buffer[i];
244     int dval = Character.digit(c, radix);
245     if (dval < 0)
246       break;
247     if (ival > max_val)
248       overflow = true;
249     else
250       ival = ival * radix + dval;
251     if (ival < 0)
252       overflow = true;
253     if (++i >= port.limit)
254       break;
255       }
256     port.pos = i;
257     return overflow ? -1 : ival;
258   }
259 
260   public String   getName() { return port.getName(); }
261   public int getLineNumber() { return port.getLineNumber(); }
262   public int getColumnNumber() { return port.getColumnNumber(); }
263 
264   public boolean isInteractive() { return interactive; }
265   public void setInteractive(boolean v) { interactive = v; }
266 
267   /** For building tokens of various kinds. */
268   public char[] tokenBuffer = new char[100];
269 
270   /** The number of chars of tokenBuffer that are used. */
271   public int tokenBufferLength = 0;
272 
273   /** Append one character to tokenBuffer, resizing it if need be. */
274   public void tokenBufferAppend(int ch)
275   {
276     if (ch > 0x10000)
277       {
278         tokenBufferAppend(((ch - 0x10000) >> 10) + 0xD800);
279         ch = (ch & 0x3FF) + 0xDC00;
280         // fall through to append low surrogate.
281       }
282     int len = tokenBufferLength;
283     char[] buffer = tokenBuffer;
284     if (len == tokenBuffer.length)
285       {
286     tokenBuffer = new char[2 * len];
287     System.arraycopy(buffer, 0, tokenBuffer, 0, len);
288     buffer = tokenBuffer;
289       }
290     buffer[len] = (char) ch;
291     tokenBufferLength = len + 1;
292   }
293 
294 
295   private int saveTokenBufferLength = -1;
296 
297   /** Start tentative parsing.  Must be followed by a reset. */
298   public void mark ()
299     throws java.io.IOException  
300   {
301     if (saveTokenBufferLength >= 0)
302       throw new Error  ("internal error: recursive call to mark not allowed");
303     port.mark(Integer.MAX_VALUE);
304     saveTokenBufferLength = tokenBufferLength;
305   }
306 
307   /** Stop tentative parsing.  Return to position where we called mark. */
308   public void reset ()
309     throws java.io.IOException  
310   {
311     if (saveTokenBufferLength < 0)
312       throw new Error  ("internal error: reset called without prior mark");
313     port.reset();
314     saveTokenBufferLength = -1;
315   }
316 }
317
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags