KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > eclipse > jdt > internal > core > dom > rewrite > TokenScanner


1 /*******************************************************************************
2  * Copyright (c) 2000, 2004 IBM Corporation and others.
3  * All rights reserved. This program and the accompanying materials
4  * are made available under the terms of the Eclipse Public License v1.0
5  * which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/epl-v10.html
7  *
8  * Contributors:
9  * IBM Corporation - initial API and implementation
10  *******************************************************************************/
11 package org.eclipse.jdt.internal.core.dom.rewrite;
12
13 import org.eclipse.core.runtime.CoreException;
14 import org.eclipse.core.runtime.IStatus;
15 import org.eclipse.core.runtime.Status;
16
17 import org.eclipse.jface.text.BadLocationException;
18 import org.eclipse.jface.text.IDocument;
19 import org.eclipse.jface.text.IRegion;
20
21 import org.eclipse.jdt.core.JavaCore;
22 import org.eclipse.jdt.core.ToolFactory;
23 import org.eclipse.jdt.core.compiler.IScanner;
24 import org.eclipse.jdt.core.compiler.ITerminalSymbols;
25 import org.eclipse.jdt.core.compiler.InvalidInputException;
26
27 /**
28  * Wraps a scanner and offers convenient methods for finding tokens
29  */
30 public class TokenScanner {
31     
32     public static final int END_OF_FILE= 20001;
33     public static final int LEXICAL_ERROR= 20002;
34     public static final int DOCUMENT_ERROR= 20003;
35     
36     private final IScanner scanner;
37     private final IDocument document;
38     private final int endPosition;
39     
40     /**
41      * Creates a TokenScanner
42      * @param scanner The scanner to be wrapped. The scanner has to support line information
43      * if the comment position methods are used.
44      */
45     public TokenScanner(IScanner scanner) {
46         this(scanner, null);
47     }
48     
49     /**
50      * Creates a TokenScanner
51      * @param scanner The scanner to be wrapped
52      * @param document The document used for line information if specified
53      */
54     public TokenScanner(IScanner scanner, IDocument document) {
55         this.scanner= scanner;
56         this.endPosition= this.scanner.getSource().length - 1;
57         this.document= document;
58     }
59     
60     /**
61      * Creates a TokenScanner
62      * @param document The textbuffer to create the scanner on
63      */
64     public TokenScanner(IDocument document) {
65         this.scanner= ToolFactory.createScanner(true, false, false, false);
66         this.scanner.setSource(document.get().toCharArray());
67         this.document= document;
68         this.endPosition= this.scanner.getSource().length - 1;
69     }
70         
71     /**
72      * Returns the wrapped scanner
73      * @return IScanner
74      */
75     public IScanner getScanner() {
76         return this.scanner;
77     }
78     
79     /**
80      * Sets the scanner offset to the given offset.
81      * @param offset The offset to set
82      */
83     public void setOffset(int offset) {
84         this.scanner.resetTo(offset, this.endPosition);
85     }
86     
87     /**
88      * @return Returns the offset after the current token
89      */
90     public int getCurrentEndOffset() {
91         return this.scanner.getCurrentTokenEndPosition() + 1;
92     }
93
94     /**
95      * @return Returns the start offset of the current token
96      */
97     public int getCurrentStartOffset() {
98         return this.scanner.getCurrentTokenStartPosition();
99     }
100     
101     /**
102      * @return Returns the length of the current token
103      */
104     public int getCurrentLength() {
105         return getCurrentEndOffset() - getCurrentStartOffset();
106     }
107
108     /**
109      * Reads the next token.
110      * @param ignoreComments If set, comments will be overread
111      * @return Return the token id.
112      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
113      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
114      */
115     public int readNext(boolean ignoreComments) throws CoreException {
116         int curr= 0;
117         do {
118             try {
119                 curr= this.scanner.getNextToken();
120                 if (curr == ITerminalSymbols.TokenNameEOF) {
121                     throw new CoreException(createError(END_OF_FILE, "End Of File", null)); //$NON-NLS-1$
122 }
123             } catch (InvalidInputException e) {
124                 throw new CoreException(createError(LEXICAL_ERROR, e.getMessage(), e)); //$NON-NLS-1$
125 }
126         } while (ignoreComments && isComment(curr));
127         return curr;
128     }
129     
130     /**
131      * Reads the next token.
132      * @param ignoreComments If set, comments will be overread.
133      * @return Return the token id.
134      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
135      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
136      */
137     private int readNextWithEOF(boolean ignoreComments) throws CoreException {
138         int curr= 0;
139         do {
140             try {
141                 curr= this.scanner.getNextToken();
142             } catch (InvalidInputException e) {
143                 throw new CoreException(createError(LEXICAL_ERROR, e.getMessage(), e)); //$NON-NLS-1$
144 }
145         } while (ignoreComments && isComment(curr));
146         return curr;
147     }
148     
149     /**
150      * Reads the next token from the given offset.
151      * @param offset The offset to start reading from.
152      * @param ignoreComments If set, comments will be overread.
153      * @return Returns the token id.
154      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
155      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
156      */
157     public int readNext(int offset, boolean ignoreComments) throws CoreException {
158         setOffset(offset);
159         return readNext(ignoreComments);
160     }
161     
162     /**
163      * Reads the next token from the given offset and returns the start offset of the token.
164      * @param offset The offset to start reading from.
165      * @param ignoreComments If set, comments will be overread
166      * @return Returns the start position of the next token.
167      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
168      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
169      */
170     public int getNextStartOffset(int offset, boolean ignoreComments) throws CoreException {
171         readNext(offset, ignoreComments);
172         return getCurrentStartOffset();
173     }
174     
175     /**
176      * Reads the next token from the given offset and returns the offset after the token.
177      * @param offset The offset to start reading from.
178      * @param ignoreComments If set, comments will be overread
179      * @return Returns the start position of the next token.
180      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
181      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
182      */
183     public int getNextEndOffset(int offset, boolean ignoreComments) throws CoreException {
184         readNext(offset, ignoreComments);
185         return getCurrentEndOffset();
186     }
187
188     /**
189      * Reads until a token is reached.
190      * @param tok The token to read to.
191      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
192      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
193      */
194     public void readToToken(int tok) throws CoreException {
195         int curr= 0;
196         do {
197             curr= readNext(false);
198         } while (curr != tok);
199     }
200
201     /**
202      * Reads until a token is reached, starting from the given offset.
203      * @param tok The token to read to.
204      * @param offset The offset to start reading from.
205      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
206      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
207      */
208     public void readToToken(int tok, int offset) throws CoreException {
209         setOffset(offset);
210         readToToken(tok);
211     }
212
213     /**
214      * Reads from the given offset until a token is reached and returns the start offset of the token.
215      * @param token The token to be found.
216      * @param startOffset The offset to start reading from.
217      * @return Returns the start position of the found token.
218      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
219      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
220      */
221     public int getTokenStartOffset(int token, int startOffset) throws CoreException {
222         readToToken(token, startOffset);
223         return getCurrentStartOffset();
224     }
225
226     /**
227      * Reads from the given offset until a token is reached and returns the offset after the token.
228      * @param token The token to be found.
229      * @param startOffset Offset to start reading from
230      * @return Returns the end position of the found token.
231      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
232      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
233      */
234     public int getTokenEndOffset(int token, int startOffset) throws CoreException {
235         readToToken(token, startOffset);
236         return getCurrentEndOffset();
237     }
238     
239     /**
240      * Reads from the given offset until a token is reached and returns the offset after the previous token.
241      * @param token The token to be found.
242      * @param startOffset The offset to start scanning from.
243      * @return Returns the end offset of the token previous to the given token.
244      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
245      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
246      */
247     public int getPreviousTokenEndOffset(int token, int startOffset) throws CoreException {
248         setOffset(startOffset);
249         int res= startOffset;
250         int curr= readNext(false);
251         while (curr != token) {
252             res= getCurrentEndOffset();
253             curr= readNext(false);
254         }
255         return res;
256     }
257     
258     /**
259      * Evaluates the start offset of comments directly ahead of a token specified by its start offset
260      *
261      * @param lastPos An offset to before the node start offset. Can be 0 but better is the end location of the previous node.
262      * @param nodeStart Start offset of the node to find the comments for.
263      * @return Returns the start offset of comments directly ahead of a token.
264      * @exception CoreException Thrown when a lexical error was detected while scanning (code LEXICAL_ERROR)
265      */
266     public int getTokenCommentStart(int lastPos, int nodeStart) throws CoreException {
267         setOffset(lastPos);
268
269         int prevEndPos= lastPos;
270         int prevEndLine= prevEndPos > 0 ? getLineOfOffset(prevEndPos - 1) : 0;
271         int nodeLine= getLineOfOffset(nodeStart);
272         
273         int res= -1;
274
275         int curr= readNextWithEOF(false);
276         int currStartPos= getCurrentStartOffset();
277         int currStartLine= getLineOfOffset(currStartPos);
278         while (curr != ITerminalSymbols.TokenNameEOF && nodeStart > currStartPos) {
279             if (TokenScanner.isComment(curr)) {
280                 int linesDifference= currStartLine - prevEndLine;
281                 if ((linesDifference > 1) || (res == -1 && (linesDifference != 0 || nodeLine == currStartLine))) {
282                     res= currStartPos; // begin new
283 }
284             } else {
285                 res= -1;
286             }
287             
288             if (curr == ITerminalSymbols.TokenNameCOMMENT_LINE) {
289                 prevEndLine= currStartLine;
290             } else {
291                 prevEndLine= getLineOfOffset(getCurrentEndOffset() - 1);
292             }
293             curr= readNextWithEOF(false);
294             currStartPos= getCurrentStartOffset();
295             currStartLine= getLineOfOffset(currStartPos);
296         }
297         if (res == -1 || curr == ITerminalSymbols.TokenNameEOF) {
298             return nodeStart;
299         }
300         if (currStartLine - prevEndLine > 1) {
301             return nodeStart;
302         }
303         return res;
304     }
305     
306     /**
307      * Looks for comments after a node and returns the end position of the comment still belonging to the node.
308      * @param nodeEnd The end position of the node
309      * @param nextTokenStart The start positoion of the next node. Optional, can be -1
310      * the line information shoould be taken from the scanner object
311      * @return Returns returns the end position of the comment still belonging to the node.
312      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
313      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
314      */
315     public int getTokenCommentEnd(int nodeEnd, int nextTokenStart) throws CoreException {
316         // assign comments to the previous comments as long they are all on the same line as the
317 // node end position or if they are on the next line but there is a separation from the next
318 // node
319 // } //aa
320 // // aa
321 //
322 // // bb
323 // public void b...
324 //
325 // } /* cc */ /*
326 // cc/*
327 // /*dd*/
328 // public void d...
329
330         int prevEndLine= getLineOfOffset(nodeEnd - 1);
331         int prevEndPos= nodeEnd;
332         int res= nodeEnd;
333         boolean sameLineComment= true;
334         
335         setOffset(nodeEnd);
336         
337         
338         int curr= readNextWithEOF(false);
339         while (curr == ITerminalSymbols.TokenNameCOMMENT_LINE || curr == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
340             int currStartLine= getLineOfOffset(getCurrentStartOffset());
341             int linesDifference= currStartLine - prevEndLine;
342
343             if (linesDifference > 1) {
344                 return prevEndPos; // separated comments
345 }
346
347             if (curr == ITerminalSymbols.TokenNameCOMMENT_LINE) {
348                 prevEndPos= getLineEnd(currStartLine);
349                 prevEndLine= currStartLine;
350             } else {
351                 prevEndPos= getCurrentEndOffset();
352                 prevEndLine= getLineOfOffset(prevEndPos - 1);
353             }
354             if (sameLineComment) {
355                 if (linesDifference == 0) {
356                     res= prevEndPos;
357                 } else {
358                     sameLineComment= false;
359                 }
360             }
361             curr= readNextWithEOF(false);
362         }
363         if (curr == ITerminalSymbols.TokenNameEOF) {
364             return prevEndPos;
365         }
366         int currStartLine= getLineOfOffset(getCurrentStartOffset());
367         int linesDifference= currStartLine - prevEndLine;
368         if (linesDifference > 1) {
369             return prevEndPos; // separated comments
370 }
371         return res;
372     }
373     
374     private int getLineOfOffset(int offset) throws CoreException {
375         if (this.document != null) {
376             try {
377                 return this.document.getLineOfOffset(offset);
378             } catch (BadLocationException e) {
379                 String JavaDoc message= "Illegal offset: " + offset; //$NON-NLS-1$
380 throw new CoreException(createError(DOCUMENT_ERROR, message, e)); //$NON-NLS-1$
381 }
382         }
383         return getScanner().getLineNumber(offset);
384     }
385     
386     private int getLineEnd(int line) throws CoreException {
387         if (this.document != null) {
388             try {
389                 IRegion region= this.document.getLineInformation(line);
390                 return region.getOffset() + region.getLength();
391             } catch (BadLocationException e) {
392                 String JavaDoc message= "Illegal line: " + line; //$NON-NLS-1$
393 throw new CoreException(createError(DOCUMENT_ERROR, message, e)); //$NON-NLS-1$
394 }
395         }
396         return getScanner().getLineEnd(line);
397     }
398     
399         
400     public static boolean isComment(int token) {
401         return token == ITerminalSymbols.TokenNameCOMMENT_BLOCK || token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC
402             || token == ITerminalSymbols.TokenNameCOMMENT_LINE;
403     }
404     
405     public static boolean isModifier(int token) {
406         switch (token) {
407             case ITerminalSymbols.TokenNamepublic:
408             case ITerminalSymbols.TokenNameprotected:
409             case ITerminalSymbols.TokenNameprivate:
410             case ITerminalSymbols.TokenNamestatic:
411             case ITerminalSymbols.TokenNamefinal:
412             case ITerminalSymbols.TokenNameabstract:
413             case ITerminalSymbols.TokenNamenative:
414             case ITerminalSymbols.TokenNamevolatile:
415             case ITerminalSymbols.TokenNamestrictfp:
416             case ITerminalSymbols.TokenNametransient:
417             case ITerminalSymbols.TokenNamesynchronized:
418                 return true;
419             default:
420                 return false;
421         }
422     }
423     
424     public static IStatus createError(int code, String JavaDoc message, Throwable JavaDoc throwable) {
425         return new Status(IStatus.ERROR, JavaCore.PLUGIN_ID, code, message, throwable);
426     }
427
428 }
429
Popular Tags