KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > netbeans > modules > java > source > save > TokenList


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */
19 package org.netbeans.modules.java.source.save;
20
21 import org.netbeans.modules.java.source.save.SourceBuffer;
22
23 import com.sun.tools.javac.parser.*;
24 import com.sun.tools.javac.parser.Scanner;
25 import com.sun.tools.javac.util.Context;
26 import com.sun.tools.javac.util.Name;
27 import com.sun.tools.javac.util.Position;
28 import java.io.*;
29 import java.util.*;
30
31 import static com.sun.tools.javac.parser.Token.*;
32
33 /**
34  * Builds a list of javac Scanner tokens for a specified source file.
35  * Look up of token indexes by source file position is supported.
36  */
37 public class TokenList extends AbstractList {
38
39     private String JavaDoc sourceFile;
40     private Token[] tokens;
41
42     /**
43      * Scan a Java source file using javac's Scanner and
44      * return its associated TokenList.
45      *
46      * @param context the context used when scanning
47      * @param sourceFile the path of the source file to be scanned
48      * @param encoding the text encoding used by this file, or null if
49      * the default encoding should be used
50      * @throws IOException if an exception occurs reading the source file
51      * @return a TokenList that represents the lexical content of the
52      * source file
53      */
54     public static TokenList scan(Context context, String JavaDoc sourceFile,
55                  SourceBuffer srcBuffer) throws IOException {
56     TokenList tlist = new TokenList(sourceFile);
57     tlist.scan(context, srcBuffer);
58     return tlist;
59     }
60
61     /**
62      * Creates a new TokenList
63      *
64      * @param sourceFile the path of the source file to be scanned.
65      */
66     private TokenList(String JavaDoc sourceFile) {
67     this.sourceFile = sourceFile;
68     }
69
70     /**
71      * Scan an InputStream to initialize this Token list.
72      */
73     protected void scan(Context context, SourceBuffer srcBuffer)
74     throws IOException {
75         Scanner.Factory factory = Scanner.Factory.instance(context);
76     Scanner scanner = factory.newScanner(srcBuffer);
77     List<Token> list = new ArrayList<Token>();
78         scanner.nextToken(); // prime the pump
79 while (scanner.token() != EOF) {
80         list.add(new Token(scanner));
81         scanner.nextToken();
82     }
83     tokens = (Token[])list.toArray(new Token[0]);
84     }
85
86     /**
87      * Returns the token at the specified index.
88      *
89      * @param index the index of the requested token.
90      */
91     public Token get(int index) {
92     return tokens[index];
93     }
94
95     /**
96      * Returns how many tokens are in this TokenList.
97      */
98     public int size() {
99     return tokens.length;
100     }
101
102     /**
103      * Returns the index of the token with the specified position (pos).
104      *
105      * @param pos the position (as defined by javac's Position class)
106      * to find
107      * @return the index of the token with that position, or -1 if
108      * no token starts at that position.
109      */
110     public int indexOf(int pos) {
111     // Since tokens are ordered by position, use a simple binary search.
112 int low = 0;
113     int high = tokens.length - 1;
114     int idx;
115     while (true) {
116         idx = (low + high) / 2;
117         if (tokens[idx].pos == pos)
118         return idx;
119         if (low > high)
120         return -1;
121         if (tokens[idx].pos < pos)
122         low = idx + 1;
123         else
124         high = idx - 1;
125     }
126     }
127
128     /**
129      * Returns the index of the token with the specified end position (endPos).
130      *
131      * @param endPos the end position (as defined by javac's Position class)
132      * to find
133      * @return the index of the token with that end position, or -1 if
134      * no token ends at that position.
135      */
136     public int indexOfEndPos(int endPos) {
137     // Since tokens are ordered by position, use a simple binary search.
138 int low = 0;
139     int high = tokens.length - 1;
140     int idx;
141     while (true) {
142         idx = (low + high) / 2;
143         if (tokens[idx].endPos == endPos)
144         return idx;
145         if (low > high)
146         return -1;
147         if (tokens[idx].endPos < endPos)
148         low = idx + 1;
149         else
150         high = idx - 1;
151     }
152     }
153
154     public String JavaDoc toString() {
155     return "TokenList of " + sourceFile + ": " +
156         tokens.length + " tokens";
157     }
158
159     public static class Token {
160     private com.sun.tools.javac.parser.Token token;
161     private int pos;
162     private int endPos;
163     private Name name;
164     private String JavaDoc value;
165     
166     private Token(Scanner scanner) {
167         token = scanner.token();
168         pos = scanner.pos();
169         endPos = scanner.endPos();
170         switch (token) {
171
172           // Literal tokens have values, but not names
173 case INTLITERAL:
174           case LONGLITERAL:
175           case FLOATLITERAL:
176           case DOUBLELITERAL:
177           case CHARLITERAL:
178           case STRINGLITERAL:
179           value = scanner.stringVal();
180           break;
181
182           // "Punctuation" tokens have neither names or values
183 case LPAREN:
184           case RPAREN:
185           case LBRACE:
186           case RBRACE:
187           case LBRACKET:
188           case RBRACKET:
189           case COMMA:
190           case SEMI:
191           case DOT:
192           break;
193
194           // All other tokens have names, but not values
195 default:
196           name = scanner.name();
197         }
198     }
199
200     public com.sun.tools.javac.parser.Token getToken() {
201         return token;
202     }
203     public int getPos() {
204         return pos;
205     }
206     public int getEndPos() {
207         return endPos;
208     }
209     public Name getName() {
210         return name;
211     }
212     public String JavaDoc getValue() {
213         return value;
214     }
215     public boolean isFlag() {
216         // from com.sun.tools.javac.code.Flags standard flags list
217 return token == PUBLIC ||
218         token == PRIVATE ||
219         token == PROTECTED ||
220         token == STATIC ||
221         token == FINAL ||
222         token == SYNCHRONIZED ||
223         token == VOLATILE ||
224         token == TRANSIENT ||
225         token == NATIVE ||
226         token == INTERFACE ||
227         token == ABSTRACT ||
228         token == STRICTFP;
229     }
230         public boolean isIdentifier() {
231             return token == IDENTIFIER;
232         }
233
234     public String JavaDoc toString() {
235         StringBuffer JavaDoc sb = new StringBuffer JavaDoc(token.name());
236         sb.append(": pos=");
237         sb.append(pos);
238         sb.append(':');
239         sb.append(pos);
240         sb.append(", endPos=");
241         sb.append(endPos);
242         sb.append(':');
243         sb.append(endPos);
244         if (name != null) {
245         sb.append(", name=");
246         sb.append(name.toString());
247         }
248         if (value != null) {
249         sb.append(", value=");
250         sb.append(value);
251         }
252         return sb.toString();
253     }
254     }
255 }
256
Popular Tags