KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > edu > rice > cs > util > sexp > Lexer


1 /*BEGIN_COPYRIGHT_BLOCK
2  *
3  * This file is part of DrJava. Download the current version of this project from http://www.drjava.org/
4  * or http://sourceforge.net/projects/drjava/
5  *
6  * DrJava Open Source License
7  *
8  * Copyright (C) 2001-2006 JavaPLT group at Rice University (javaplt@rice.edu). All rights reserved.
9  *
10  * Developed by: Java Programming Languages Team, Rice University, http://www.cs.rice.edu/~javaplt/
11  *
12  * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
13  * documentation files (the "Software"), to deal with the Software without restriction, including without limitation
14  * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
15  * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
16  *
17  * - Redistributions of source code must retain the above copyright notice, this list of conditions and the
18  * following disclaimers.
19  * - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
20  * following disclaimers in the documentation and/or other materials provided with the distribution.
21  * - Neither the names of DrJava, the JavaPLT, Rice University, nor the names of its contributors may be used to
22  * endorse or promote products derived from this Software without specific prior written permission.
23  * - Products derived from this software may not be called "DrJava" nor use the term "DrJava" as part of their
24  * names without prior written permission from the JavaPLT group. For permission, write to javaplt@rice.edu.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
27  * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28  * CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
29  * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
30  * WITH THE SOFTWARE.
31  *
32  *END_COPYRIGHT_BLOCK*/
33
34 package edu.rice.cs.util.sexp;
35  
36 import java.util.*;
37 import java.io.*;
38
39 class Lexer extends StreamTokenizer {
40   
41   public HashMap<String JavaDoc,Tokens.SExpToken> wordTable = new HashMap<String JavaDoc,Tokens.SExpToken>();
42   
43   private Tokens.SExpToken buffer;
44   
45   public Lexer(File file) throws FileNotFoundException{
46     this(new BufferedReader(new FileReader(file)));
47   }
48   
49   public Lexer(Reader reader) {
50     super(new BufferedReader(reader));
51     initLexer();
52   }
53   
54   private void initLexer() {
55     
56     // configure StreamTokenizer portion of this
57 resetSyntax();
58     parseNumbers();
59     slashSlashComments(true);
60     wordChars('!','\'');
61     wordChars('*','~');
62     quoteChar('"');
63     ordinaryChars('(',')');
64     whitespaceChars(0,' ');
65     commentChar(';');
66     
67     initWordTable();
68     buffer = null; // buffer initially empty
69 }
70   
71   /** Skips through the input stream until an EOL is encountered */
72   public void flush() throws IOException {
73     eolIsSignificant(true);
74     while (nextToken() != TT_EOL) ; // eat tokens until EOL
75 eolIsSignificant(false);
76   }
77     
78   /** Performs a nextToken() operation from StreamTokenizer except
79    * for throwing an unchecked LexingException instead of a checked IOException */
80   private int getToken() {
81     try {
82       int tokenType = nextToken();
83       return tokenType;
84     } catch(IOException e) {
85       throw new LexingException("Unable to read the data from the given input");
86     }
87   }
88   
89   /** Returns the next Tokens.SExpToken without consuming it */
90   public Tokens.SExpToken peek() {
91     if (buffer == null) buffer = readToken();
92     return buffer;
93   }
94   
95   /** Reads the next Tokens.SExpToken from the input stream and consumes it;
96    * Returns the Tokens.SExpToken object representing this Tokens.SExpToken */
97   public Tokens.SExpToken readToken() {
98     
99     if (buffer != null) {
100       Tokens.SExpToken token = buffer;
101       buffer = null; // clear buffer
102 return token;
103     }
104     
105     int tokenType = getToken();
106     // Process the Tokens.SExpToken returned by StreamTokenizer
107 switch (tokenType) {
108       case TT_NUMBER:
109         return new Tokens.NumberToken(nval);
110         
111       case TT_WORD:
112         String JavaDoc s = sval.toLowerCase();
113         Tokens.SExpToken regToken = wordTable.get(s);
114         if (regToken == null) return new Tokens.WordToken(sval);
115         
116         return regToken;
117         
118       case TT_EOF: return null;
119       case '(': return Tokens.LeftParenToken.ONLY;
120       case ')': return Tokens.RightParenToken.ONLY;
121       case '"': return new Tokens.QuotedTextToken(sval);
122       case '\\':
123 // int t = getToken();
124 // if (t == '"') {
125 // return new Tokens.WordToken("\"");
126 // }
127 // else if (t == '\\') {
128 // return new Tokens.WordToken("\\");
129 // }
130 // else if (t == ' ') {
131 // return new Tokens.WordToken(" ");
132 // }
133 // else if (t == 'n') {
134 // return new Tokens.WordToken("\n");
135 // }
136 // else if (t == 't') {
137 // return new Tokens.WordToken("\t");
138 // }
139 // else {
140 // pushBack();
141 return Tokens.BackSlashToken.ONLY;
142 // throw new SExpParseException("Invalid escape sequence: \\" + (char)t);
143
144       default:
145         return new Tokens.WordToken("" + (char)tokenType);
146     }
147   }
148   
149   
150   /** Initialize the word table used by the lexer to classify Tokens */
151   private void initWordTable() {
152     // initialize wordTable
153 wordTable.put("true", Tokens.BooleanToken.TRUE);
154     wordTable.put("false", Tokens.BooleanToken.FALSE);
155   }
156 }
Popular Tags