KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > netbeans > lib > lexer > test > simple > SimpleLexer


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */
19
20 package org.netbeans.lib.lexer.test.simple;
21
22 import java.util.HashMap JavaDoc;
23 import java.util.Map JavaDoc;
24 import org.netbeans.api.lexer.Token;
25 import org.netbeans.spi.lexer.Lexer;
26 import org.netbeans.spi.lexer.LexerInput;
27 import org.netbeans.spi.lexer.LexerRestartInfo;
28 import org.netbeans.spi.lexer.TokenFactory;
29
30 /**
31  * Simple implementation a lexer.
32  *
33  * @author mmetelka
34  */
35 final class SimpleLexer implements Lexer<SimpleTokenId> {
36
37     // Copy of LexerInput.EOF
38 private static final int EOF = LexerInput.EOF;
39
40     private static final Map JavaDoc<String JavaDoc,SimpleTokenId> keywords = new HashMap JavaDoc<String JavaDoc,SimpleTokenId>();
41     static {
42         keywords.put(SimpleTokenId.PUBLIC.fixedText(), SimpleTokenId.PUBLIC);
43         keywords.put(SimpleTokenId.PRIVATE.fixedText(), SimpleTokenId.PRIVATE);
44         keywords.put(SimpleTokenId.STATIC.fixedText(), SimpleTokenId.STATIC);
45     }
46     
47     
48     private final LexerInput input;
49     
50     private final TokenFactory<SimpleTokenId> tokenFactory;
51     
52     private final int version;
53
54     SimpleLexer(LexerRestartInfo<SimpleTokenId> info) {
55         this.input = info.input();
56         this.tokenFactory = info.tokenFactory();
57         assert (info.state() == null); // never set to non-null value in state()
58
59         Integer JavaDoc ver = (Integer JavaDoc)info.getAttributeValue("version");
60         this.version = (ver != null) ? ver.intValue() : 2;
61     }
62
63     public Object JavaDoc state() {
64         return null; // always in default state after token recognition
65 }
66
67     public Token<SimpleTokenId> nextToken() {
68         while (true) {
69             int c = input.read();
70             switch (c) {
71                 case '"': // string literal
72 while (true) {
73                         switch (input.read()) {
74                             case '"': // NOI18N
75 return token(SimpleTokenId.STRING_LITERAL);
76                             case '\\':
77                                 input.read(); // skip the next character
78 break;
79                             case '\r': input.consumeNewline();
80                             case '\n':
81                             case EOF:
82                                 return token(SimpleTokenId.STRING_LITERAL_INCOMPLETE);
83                         }
84                     }
85
86                 case '+':
87                     switch (input.read()) {
88                         case '-':
89                             if (input.read() == '+')
90                                 return token(SimpleTokenId.PLUS_MINUS_PLUS);
91                             input.backup(2);
92                             return token(SimpleTokenId.PLUS);
93
94                     }
95                     input.backup(1);
96                     return token(SimpleTokenId.PLUS);
97
98                 case '-':
99                     return token(SimpleTokenId.MINUS);
100
101                 case '*':
102                     return token(SimpleTokenId.STAR);
103                             
104                 case '/':
105                     switch (input.read()) {
106                         case '/': // in single-line comment
107 while (true)
108                                 switch (input.read()) {
109                                     case '\r': input.consumeNewline();
110                                     case '\n':
111                                     case EOF:
112                                         return token(SimpleTokenId.LINE_COMMENT);
113                                 }
114                         case '*': // in multi-line or javadoc comment
115 c = input.read();
116                             if (c == '*') { // either javadoc comment or empty multi-line comment /**/
117 c = input.read();
118                                     if (c == '/')
119                                         return token(SimpleTokenId.BLOCK_COMMENT);
120                                     while (true) { // in javadoc comment
121 while (c == '*') {
122                                             c = input.read();
123                                             if (c == '/')
124                                                 return token(SimpleTokenId.JAVADOC_COMMENT);
125                                             else if (c == EOF)
126                                                 return token(SimpleTokenId.JAVADOC_COMMENT_INCOMPLETE);
127                                         }
128                                         if (c == EOF)
129                                             return token(SimpleTokenId.JAVADOC_COMMENT_INCOMPLETE);
130                                         c = input.read();
131                                     }
132
133                             } else { // in multi-line comment (and not after '*')
134 while (true) {
135                                     c = input.read();
136                                     while (c == '*') {
137                                         c = input.read();
138                                         if (c == '/')
139                                             return token(SimpleTokenId.BLOCK_COMMENT);
140                                         else if (c == EOF)
141                                             return token(SimpleTokenId.BLOCK_COMMENT_INCOMPLETE);
142                                     }
143                                     if (c == EOF)
144                                         return token(SimpleTokenId.BLOCK_COMMENT_INCOMPLETE);
145                                 }
146                             }
147                     }
148                     input.backup(1);
149                     return token(SimpleTokenId.DIV);
150
151                 // All Character.isWhitespace(c) below 0x80 follow
152 // ['\t' - '\r'] and [0x1c - ' ']
153 case '\t':
154                 case '\n':
155                 case 0x0b:
156                 case '\f':
157                 case '\r':
158                 case 0x1c:
159                 case 0x1d:
160                 case 0x1e:
161                 case 0x1f:
162                     return finishWhitespace();
163                 case ' ':
164                     c = input.read();
165                     if (c == EOF || !Character.isWhitespace(c)) { // Return single space as flyweight token
166 input.backup(1);
167                         return tokenFactory.getFlyweightToken(SimpleTokenId.WHITESPACE, " ");
168                     }
169                     return finishWhitespace();
170
171                 case EOF: // no more chars on the input
172 return null; // the only legal situation when null can be returned
173
174                 default:
175                     if (Character.isJavaIdentifierStart((char)c))
176                         return finishIdentifier();
177                     if (c >= 0x80 && Character.isWhitespace((char)c))
178                         return finishWhitespace();
179
180                     // Invalid char
181 return token(SimpleTokenId.ERROR);
182             }
183         }
184     }
185         
186     private Token<SimpleTokenId> finishWhitespace() {
187         while (true) {
188             int c = input.read();
189             if (c == EOF || !Character.isWhitespace(c)) {
190                 input.backup(1);
191                 return tokenFactory.createToken(SimpleTokenId.WHITESPACE);
192             }
193         }
194     }
195     
196     private Token<SimpleTokenId> finishIdentifier() {
197         while (true) {
198             int c = input.read();
199             if (c == EOF || !Character.isJavaIdentifierPart(c)) {
200                 input.backup(1);
201                 // Check whether the identifier is not a keyword
202 SimpleTokenId id = keywords.get(input.readText());
203                 // Test: Only recognize STATIC keyword in version >= 2
204 if (id == SimpleTokenId.STATIC && version < 2) {
205                     id = null; // force IDENTIFIER
206 }
207                 return (id == null)
208                         ? tokenFactory.createToken(SimpleTokenId.IDENTIFIER)
209                         : token(id);
210             }
211         }
212     }
213
214     private Token<SimpleTokenId> token(SimpleTokenId id) {
215         String JavaDoc fixedText = id.fixedText();
216         return (fixedText != null)
217                 ? tokenFactory.getFlyweightToken(id, fixedText)
218                 : tokenFactory.createToken(id);
219     }
220     
221     public void release() {
222     }
223
224 }
225
Popular Tags