TokenScanner


1   /*******************************************************************************
2    * Copyright (c) 2000, 2006 IBM Corporation and others.
3    * All rights reserved. This program and the accompanying materials
4    * are made available under the terms of the Eclipse Public License v1.0
5    * which accompanies this distribution, and is available at
6    * http://www.eclipse.org/legal/epl-v10.html
7    *
8    * Contributors:
9    *     IBM Corporation - initial API and implementation
10   *******************************************************************************/
11  package org.eclipse.jdt.internal.corext.dom;
12  
13  import org.eclipse.core.runtime.CoreException;
14  import org.eclipse.core.runtime.IStatus;
15  
16  import org.eclipse.jface.text.BadLocationException;
17  import org.eclipse.jface.text.IDocument;
18  import org.eclipse.jface.text.IRegion;
19  
20  import org.eclipse.jdt.core.ICompilationUnit;
21  import org.eclipse.jdt.core.IJavaProject;
22  import org.eclipse.jdt.core.JavaCore;
23  import org.eclipse.jdt.core.JavaModelException;
24  import org.eclipse.jdt.core.ToolFactory;
25  import org.eclipse.jdt.core.compiler.IScanner;
26  import org.eclipse.jdt.core.compiler.ITerminalSymbols;
27  import org.eclipse.jdt.core.compiler.InvalidInputException;
28  
29  import org.eclipse.jdt.internal.ui.JavaUIStatus;
30  
31  /**
32   * Wraps a scanner and offers convenient methods for finding tokens 
33   */
34  public class TokenScanner {
35      
36      public static final int END_OF_FILE= 20001;
37      public static final int LEXICAL_ERROR= 20002;
38      public static final int DOCUMENT_ERROR= 20003;
39      
40      private IScanner fScanner;
41      private IDocument fDocument;
42      private int fEndPosition;
43      
44      /**
45       * Creates a TokenScanner
46       * @param scanner The scanner to be wrapped. The scanner has to support line information
47       * if the comment position methods are used.
48       */
49      public TokenScanner(IScanner scanner) {
50          this(scanner, null);
51      }
52      
53      /**
54       * Creates a TokenScanner
55       * @param scanner The scanner to be wrapped
56       * @param document The document used for line information if specified
57       */
58      public TokenScanner(IScanner scanner, IDocument document) {
59          fScanner= scanner;
60          fEndPosition= fScanner.getSource().length - 1;
61          fDocument= document;
62      }
63      
64      /**
65       * Creates a TokenScanner
66       * @param document The textbuffer to create the scanner on
67       */
68      public TokenScanner(IDocument document, IJavaProject project) {
69          String   sourceLevel= project.getOption(JavaCore.COMPILER_SOURCE, true);
70          String   complianceLevel= project.getOption(JavaCore.COMPILER_COMPLIANCE, true);
71          fScanner= ToolFactory.createScanner(true, false, false, sourceLevel, complianceLevel); // no line info required
72          fScanner.setSource(document.get().toCharArray());
73          fDocument= document;
74          fEndPosition= fScanner.getSource().length - 1;
75      }
76      
77      /**
78       * Creates a TokenScanner
79       * @param cu The compliation unit to can on
80       * @throws JavaModelException thorwn if the buffer cannot be accessed
81       */
82      public TokenScanner(ICompilationUnit cu) throws JavaModelException {
83          IJavaProject project= cu.getJavaProject();
84          String   sourceLevel= project.getOption(JavaCore.COMPILER_SOURCE, true);
85          String   complianceLevel= project.getOption(JavaCore.COMPILER_COMPLIANCE, true);
86          fScanner= ToolFactory.createScanner(true, false, true, sourceLevel, complianceLevel); // line info required
87          fScanner.setSource(cu.getBuffer().getCharacters());
88          fDocument= null; // use scanner for line information
89          fEndPosition= fScanner.getSource().length - 1;
90      }   
91          
92      /**
93       * Returns the wrapped scanner
94       * @return IScanner
95       */
96      public IScanner getScanner() {
97          return fScanner;
98      }
99      
100     /**
101      * Sets the scanner offset to the given offset.
102      * @param offset The offset to set
103      */
104     public void setOffset(int offset) {
105         fScanner.resetTo(offset, fEndPosition);
106     }
107     
108     /**
109      * @return Returns the offset after the current token
110      */ 
111     public int getCurrentEndOffset() {
112         return fScanner.getCurrentTokenEndPosition() + 1;
113     }
114 
115     /**
116      * @return Returns the start offset of the current token
117      */     
118     public int getCurrentStartOffset() {
119         return fScanner.getCurrentTokenStartPosition();
120     }
121     
122     /**
123      * @return Returns the length of the current token
124      */ 
125     public int getCurrentLength() {
126         return getCurrentEndOffset() - getCurrentStartOffset();
127     }   
128 
129     /**
130      * Reads the next token.
131      * @param ignoreComments If set, comments will be overread
132      * @return Return the token id.
133      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
134      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
135      */
136     public int readNext(boolean ignoreComments) throws CoreException {
137         int curr= 0;
138         do {
139             try {
140                 curr= fScanner.getNextToken();
141                 if (curr == ITerminalSymbols.TokenNameEOF) {
142                     throw new CoreException(createError(END_OF_FILE, "End Of File", null)); //$NON-NLS-1$
143                 }
144             } catch (InvalidInputException e) {
145                 throw new CoreException(createError(LEXICAL_ERROR, e.getMessage(), e)); 
146             }
147         } while (ignoreComments && isComment(curr));
148         return curr;
149     }
150     
151     /**
152      * Reads the next token.
153      * @param ignoreComments If set, comments will be overread.
154      * @return Return the token id.
155      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
156      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
157      */
158     private int readNextWithEOF(boolean ignoreComments) throws CoreException {
159         int curr= 0;
160         do {
161             try {
162                 curr= fScanner.getNextToken();
163             } catch (InvalidInputException e) {
164                 throw new CoreException(createError(LEXICAL_ERROR, e.getMessage(), e)); 
165             }
166         } while (ignoreComments && isComment(curr));
167         return curr;
168     }   
169     
170     /**
171      * Reads the next token from the given offset.
172      * @param offset The offset to start reading from.
173      * @param ignoreComments If set, comments will be overread.
174      * @return Returns the token id.
175      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
176      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
177      */     
178     public int readNext(int offset, boolean ignoreComments) throws CoreException {
179         setOffset(offset);
180         return readNext(ignoreComments);
181     }
182     
183     /**
184      * Reads the next token from the given offset and returns the start offset of the token.
185      * @param offset The offset to start reading from.
186      * @param ignoreComments If set, comments will be overread
187      * @return Returns the start position of the next token. 
188      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
189      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
190      */ 
191     public int getNextStartOffset(int offset, boolean ignoreComments) throws CoreException {
192         readNext(offset, ignoreComments);
193         return getCurrentStartOffset();
194     }
195     
196     /**
197      * Reads the next token from the given offset and returns the offset after the token.
198      * @param offset The offset to start reading from.
199      * @param ignoreComments If set, comments will be overread
200      * @return Returns the start position of the next token. 
201      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
202      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
203      */     
204     public int getNextEndOffset(int offset, boolean ignoreComments) throws CoreException {
205         readNext(offset, ignoreComments);
206         return getCurrentEndOffset();
207     }       
208 
209     /**
210      * Reads until a token is reached.
211      * @param tok The token to read to.
212      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
213      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
214      */
215     public void readToToken(int tok) throws CoreException {
216         int curr= 0;
217         do {
218             curr= readNext(false);
219         } while (curr != tok); 
220     }
221 
222     /**
223      * Reads until a token is reached, starting from the given offset.
224      * @param tok The token to read to.
225      * @param offset The offset to start reading from.
226      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
227      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
228      */         
229     public void readToToken(int tok, int offset) throws CoreException {
230         setOffset(offset);
231         readToToken(tok);
232     }
233 
234     /**
235      * Reads from the given offset until a token is reached and returns the start offset of the token.
236      * @param token The token to be found.
237      * @param startOffset The offset to start reading from.
238      * @return Returns the start position of the found token. 
239      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
240      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
241      */ 
242     public int getTokenStartOffset(int token, int startOffset) throws CoreException {
243         readToToken(token, startOffset);
244         return getCurrentStartOffset();
245     }   
246 
247     /**
248      * Reads from the given offset until a token is reached and returns the offset after the token.
249      * @param token The token to be found.
250      * @param startOffset Offset to start reading from
251      * @return Returns the end position of the found token. 
252      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
253      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
254      */     
255     public int getTokenEndOffset(int token, int startOffset) throws CoreException {
256         readToToken(token, startOffset);
257         return getCurrentEndOffset();
258     }
259     
260     /**
261      * Reads from the given offset until a token is reached and returns the offset after the previous token.
262      * @param token The token to be found.
263      * @param startOffset The offset to start scanning from.
264      * @return Returns the end offset of the token previous to the given token.
265      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
266      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
267      */     
268     public int getPreviousTokenEndOffset(int token, int startOffset) throws CoreException {
269         setOffset(startOffset);
270         int res= startOffset;
271         int curr= readNext(false);
272         while (curr != token) {
273             res= getCurrentEndOffset();
274             curr= readNext(false);
275         }
276         return res;
277     }
278     
279     /**
280      * Evaluates the start offset of comments directly ahead of a token specified by its start offset
281      * 
282      * @param lastPos An offset to before the node start offset. Can be 0 but better is the end location of the previous node. 
283      * @param nodeStart Start offset of the node to find the comments for.
284      * @return Returns the start offset of comments directly ahead of a token.
285      * @exception CoreException Thrown when a lexical error was detected while scanning (code LEXICAL_ERROR)
286      */     
287     public int getTokenCommentStart(int lastPos, int nodeStart) throws CoreException {
288         setOffset(lastPos);
289 
290         int prevEndPos= lastPos;
291         int prevEndLine= prevEndPos > 0 ? getLineOfOffset(prevEndPos - 1) : 0;
292         int nodeLine= getLineOfOffset(nodeStart);
293         
294         int res= -1;
295 
296         int curr= readNextWithEOF(false);
297         int currStartPos= getCurrentStartOffset();
298         int currStartLine= getLineOfOffset(currStartPos);
299         while (curr != ITerminalSymbols.TokenNameEOF && nodeStart > currStartPos) {
300             if (TokenScanner.isComment(curr)) {
301                 int linesDifference= currStartLine - prevEndLine;
302                 if ((linesDifference > 1) || (res == -1 && (linesDifference != 0 || nodeLine == currStartLine))) {
303                     res= currStartPos; // begin new
304                 }
305             } else {
306                 res= -1;
307             }
308             
309             if (curr == ITerminalSymbols.TokenNameCOMMENT_LINE) {
310                 prevEndLine= currStartLine;
311             } else {
312                 prevEndLine= getLineOfOffset(getCurrentEndOffset() - 1);
313             }                   
314             curr= readNextWithEOF(false);
315             currStartPos= getCurrentStartOffset();
316             currStartLine= getLineOfOffset(currStartPos);
317         }
318         if (res == -1 || curr == ITerminalSymbols.TokenNameEOF) {
319             return nodeStart;
320         }
321         if (currStartLine - prevEndLine > 1) {
322             return nodeStart;
323         }           
324         return res;
325     }
326     
327     /**
328      * Looks for comments after a node and returns the end position of the comment still belonging to the node.
329      * @param nodeEnd The end position of the node
330      * @param nextTokenStart The start positoion of the next node. Optional, can be -1
331      * the line information shoould be taken from the scanner object
332      * @return Returns returns the end position of the comment still belonging to the node.
333      * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
334      * or a lexical error was detected while scanning (code LEXICAL_ERROR)
335      */     
336     public int getTokenCommentEnd(int nodeEnd, int nextTokenStart) throws CoreException {
337         // assign comments to the previous comments as long they are all on the same line as the
338         // node end position or if they are on the next line but there is a separation from the next
339         // node
340         // } //aa
341         // // aa
342         //
343         // // bb
344         // public void b...
345         //
346         // } /* cc */ /*
347         // cc/*
348         // /*dd*/
349         // public void d...
350         
351         int prevEndLine= getLineOfOffset(nodeEnd - 1);
352         int prevEndPos= nodeEnd;
353         int res= nodeEnd;
354         boolean sameLineComment= true;
355         
356         setOffset(nodeEnd);
357         
358         
359         int curr= readNextWithEOF(false);
360         while (curr == ITerminalSymbols.TokenNameCOMMENT_LINE || curr == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
361             int currStartLine= getLineOfOffset(getCurrentStartOffset());
362             int linesDifference= currStartLine - prevEndLine;
363 
364             if (linesDifference > 1) {
365                 return prevEndPos; // separated comments
366             }
367 
368             if (curr == ITerminalSymbols.TokenNameCOMMENT_LINE) {
369                 prevEndPos= getLineEnd(currStartLine);
370                 prevEndLine= currStartLine;
371             } else {
372                 prevEndPos= getCurrentEndOffset();
373                 prevEndLine= getLineOfOffset(prevEndPos - 1);
374             }
375             if (sameLineComment) {
376                 if (linesDifference == 0) {
377                     res= prevEndPos;
378                 } else {
379                     sameLineComment= false;
380                 }
381             }
382             curr= readNextWithEOF(false);
383         }
384         if (curr == ITerminalSymbols.TokenNameEOF) {
385             return prevEndPos;
386         }
387         int currStartLine= getLineOfOffset(getCurrentStartOffset());
388         int linesDifference= currStartLine - prevEndLine;
389         if (linesDifference > 1) {
390             return prevEndPos; // separated comments
391         }
392         return res;
393     }
394     
395     public int getLineOfOffset(int offset) throws CoreException {
396         if (fDocument != null) {
397             try {
398                 return fDocument.getLineOfOffset(offset);
399             } catch (BadLocationException e) {
400                 String   message= "Illegal offset: " + offset; //$NON-NLS-1$
401                 throw new CoreException(createError(DOCUMENT_ERROR, message, e)); 
402             }
403         }
404         return getScanner().getLineNumber(offset);
405     }
406     
407     public int getLineEnd(int line) throws CoreException {
408         if (fDocument != null) {
409             try {
410                 IRegion region= fDocument.getLineInformation(line);
411                 return region.getOffset() + region.getLength();
412             } catch (BadLocationException e) {
413                 String   message= "Illegal line: " + line; //$NON-NLS-1$
414                 throw new CoreException(createError(DOCUMENT_ERROR, message, e)); 
415             }
416         }
417         return getScanner().getLineEnd(line);
418     }           
419 
420     public static boolean isComment(int token) {
421         return token == ITerminalSymbols.TokenNameCOMMENT_BLOCK || token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC 
422             || token == ITerminalSymbols.TokenNameCOMMENT_LINE;
423     }
424     
425     public static boolean isModifier(int token) {
426         switch (token) {
427             case ITerminalSymbols.TokenNamepublic:
428             case ITerminalSymbols.TokenNameprotected:
429             case ITerminalSymbols.TokenNameprivate:
430             case ITerminalSymbols.TokenNamestatic:
431             case ITerminalSymbols.TokenNamefinal:
432             case ITerminalSymbols.TokenNameabstract:
433             case ITerminalSymbols.TokenNamenative:
434             case ITerminalSymbols.TokenNamevolatile:
435             case ITerminalSymbols.TokenNamestrictfp:
436             case ITerminalSymbols.TokenNametransient:
437             case ITerminalSymbols.TokenNamesynchronized:
438                 return true;
439             default:
440                 return false;
441         }
442     }
443     
444     private IStatus createError(int code, String   message, Throwable   e) {
445         return JavaUIStatus.createError(DOCUMENT_ERROR, message, e);
446         //      return new Status(IStatus.ERROR, JavaCore.PLUGIN_ID, code, message, throwable);
447     }
448 
449 }
450
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags