KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > netbeans > lib > lexer > test > dump > TokenDumpLexer


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */
19
20 package org.netbeans.lib.lexer.test.dump;
21
22 import org.netbeans.api.lexer.PartType;
23 import org.netbeans.api.lexer.Token;
24 import org.netbeans.spi.lexer.Lexer;
25 import org.netbeans.spi.lexer.LexerInput;
26 import org.netbeans.spi.lexer.LexerRestartInfo;
27 import org.netbeans.spi.lexer.TokenFactory;
28 import org.netbeans.spi.lexer.TokenPropertyProvider;
29 import org.netbeans.spi.lexer.TokenPropertyProvider;
30
31 /**
32  * Simple implementation a lexer.
33  *
34  * @author mmetelka
35  */
36 final class TokenDumpLexer implements Lexer<TokenDumpTokenId> {
37
38     // Copy of LexerInput.EOF
39 private static final int EOF = LexerInput.EOF;
40     
41     private LexerInput input;
42
43     private TokenFactory<TokenDumpTokenId> tokenFactory;
44     
45     TokenDumpLexer(LexerRestartInfo<TokenDumpTokenId> info) {
46         this.input = info.input();
47         this.tokenFactory = info.tokenFactory();
48     }
49
50     public Object JavaDoc state() {
51         return null;
52     }
53
54     public Token<TokenDumpTokenId> nextToken() {
55         int c = input.read();
56         switch (c) {
57             case '\r':
58                 input.consumeNewline(); // continue to '\n' handling
59 case '\n': // newline
60 return token(TokenDumpTokenId.NEWLINE);
61
62             case EOF:
63                 return null;
64
65             case '.':
66                 switch (c = input.read()) {
67                     case '\\': // ".\"
68 if ((c = input.read()) == '.') { // ".\."
69 switch (c = input.read()) {
70                                 case 'b':
71                                     return finishCharLiteralOrText(TokenDumpTokenId.BACKSPACE_CHAR, '\b');
72                                 case 'f':
73                                     return finishCharLiteralOrText(TokenDumpTokenId.FORM_FEED_CHAR, '\f');
74                                 case 't':
75                                     return finishCharLiteralOrText(TokenDumpTokenId.TAB_CHAR, '\t');
76                                 case 'r':
77                                     return finishCharLiteralOrText(TokenDumpTokenId.CR_CHAR, '\r');
78                                 case 'n':
79                                     return finishCharLiteralOrText(TokenDumpTokenId.NEWLINE_CHAR, '\n');
80                                 case 'u':
81                                     if ((c = input.read()) == '.')
82                                         return finishUnicodeOrText();
83                             }
84                         }
85                         input.backup(1);
86                         return finishText();
87                             
88                     case 'e': // ".e"
89 if ((c = input.read()) == '.'
90                          && (c = input.read()) == 'o'
91                          && (c = input.read()) == '.'
92                          && (c = input.read()) == 'f'
93                          && (c = input.read()) == '.'
94                         )
95                             return finishNewlineOrText(TokenDumpTokenId.EOF_VIRTUAL);
96                         input.backup(1);
97                         return finishText();
98                         
99                     case 't': // ".t"
100 if ((c = input.read()) == '.'
101                          && (c = input.read()) == 'e'
102                          && (c = input.read()) == '.'
103                          && (c = input.read()) == 's'
104                          && (c = input.read()) == '.'
105                          && (c = input.read()) == 't'
106                          && (c = input.read()) == '.'
107                         ) { // ".t.e.s.t."
108 return finishTillNewline(TokenDumpTokenId.TEST_NAME);
109                         }
110                         input.backup(1);
111                         return finishText();
112                         
113                     case EOF:
114                         return token(TokenDumpTokenId.TEXT);
115                 }
116
117             default:
118                 return finishText();
119         }
120     }
121     
122     private Token<TokenDumpTokenId> finishText() {
123         return finishTillNewline(TokenDumpTokenId.TEXT);
124     }
125
126     private Token<TokenDumpTokenId> finishTillNewline(TokenDumpTokenId id) {
127         while (true) {
128             switch (input.read()) {
129                 case '\r':
130                 case '\n':
131                 case EOF:
132                     input.backup(1);
133                     return token(id);
134             }
135         }
136     }
137     
138     private Token<TokenDumpTokenId> finishNewlineOrText(TokenDumpTokenId id) {
139         // If newline follows then return the given id otherwise return text
140 switch (input.read()) {
141             case '\r':
142             case '\n':
143             case EOF: // EOF is also valid ending
144 input.backup(1);
145                 return token(id);
146         }
147         return finishText();
148     }
149     
150     private Token<TokenDumpTokenId> finishUnicodeOrText() {
151         // If 4 unicode hex numbers followed by '.' return UNICODE_CHAR otherwise TEXT
152 int c;
153         int number = 0;
154         int hexDigit = 0;
155         for (int i = 4; i > 0; i--) { // read 4 unicode digits
156 switch (c = input.read()) {
157                 case '0': case '1': case '2': case '3': case '4':
158                 case '5': case '6': case '7': case '8': case '9':
159                     hexDigit = c - '0';
160                     break;
161                 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
162                     hexDigit = c - 'a' + 10;
163                     break;
164                 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
165                     hexDigit = c - 'A' + 10;
166                     break;
167                 default:
168                     input.backup(1);
169                     return finishText();
170                     
171             }
172             number = (number << 4) | hexDigit;
173         }
174         return finishCharLiteralOrText(TokenDumpTokenId.UNICODE_CHAR, (char)number);
175     }
176     
177     private Token<TokenDumpTokenId> finishCharLiteralOrText(TokenDumpTokenId id, char ch) {
178         int c;
179         if ((c = input.read()) == '.') {
180             switch (c = input.read()) {
181                 case '\r':
182                 case '\n':
183                 case EOF:
184                     input.backup(1);
185                     return tokenFactory.createPropertyToken(id, input.readLength(),
186                         new UnicodeCharValueProvider(new Character JavaDoc(ch)), PartType.COMPLETE);
187             }
188         }
189         input.backup(1);
190         return finishText();
191     }
192     
193     private Token<TokenDumpTokenId> token(TokenDumpTokenId id) {
194         return tokenFactory.createToken(id);
195     }
196     
197     public void release() {
198     }
199
200     private static final class UnicodeCharValueProvider implements TokenPropertyProvider {
201         
202         private Character JavaDoc ch;
203         
204         UnicodeCharValueProvider(Character JavaDoc ch) {
205             this.ch = ch;
206         }
207         
208         public Object JavaDoc getValue(Token token, Object JavaDoc key) {
209             if (TokenDumpTokenId.UNICODE_CHAR_TOKEN_PROPERTY.equals(key))
210                 return ch;
211             return null; // no non-tokenStore value
212 }
213         
214     }
215     
216 }
217
Popular Tags