KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > eclipse > jdt > internal > ui > compare > JavaTokenComparator


1 /*******************************************************************************
2  * Copyright (c) 2000, 2007 IBM Corporation and others.
3  * All rights reserved. This program and the accompanying materials
4  * are made available under the terms of the Eclipse Public License v1.0
5  * which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/epl-v10.html
7  *
8  * Contributors:
9  * IBM Corporation - initial API and implementation
10  *******************************************************************************/
11 package org.eclipse.jdt.internal.ui.compare;
12
13 import org.eclipse.core.runtime.Assert;
14
15 import org.eclipse.compare.contentmergeviewer.ITokenComparator;
16 import org.eclipse.compare.rangedifferencer.IRangeComparator;
17
18 import org.eclipse.jdt.core.ToolFactory;
19 import org.eclipse.jdt.core.compiler.IScanner;
20 import org.eclipse.jdt.core.compiler.ITerminalSymbols;
21 import org.eclipse.jdt.core.compiler.InvalidInputException;
22
23 import org.eclipse.jdt.internal.corext.dom.TokenScanner;
24
25
26 /**
27  * A comparator for Java tokens.
28  */
29 public class JavaTokenComparator implements ITokenComparator {
30     
31     /**
32      * Factory to create text token comparators.
33      * This is a workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=183224 .
34      */
35     public static interface ITokenComparatorFactory {
36         /**
37          * @param text text to be tokenized
38          * @return a token comparator
39          */
40         public ITokenComparator createTokenComparator(String JavaDoc text);
41     }
42         
43     private static final boolean DEBUG= false;
44     
45     private final String JavaDoc fText;
46     private final ITokenComparatorFactory fTextTokenComparatorFactory;
47     private int fCount;
48     private int[] fStarts;
49     private int[] fLengths;
50
51     /**
52      * Creates a token comparator for the given string.
53      *
54      * @param text the text to be tokenized
55      */
56     public JavaTokenComparator(String JavaDoc text) {
57         this(text, null);
58     }
59     
60     /**
61      * Creates a token comparator for the given string.
62      *
63      * @param text the text to be tokenized
64      * @param textTokenComparatorFactory a factory to create text token comparators
65      */
66     public JavaTokenComparator(String JavaDoc text, ITokenComparatorFactory textTokenComparatorFactory) {
67         
68         fTextTokenComparatorFactory= textTokenComparatorFactory;
69         Assert.isLegal(text != null);
70         
71         fText= text;
72         
73         int length= fText.length();
74         fStarts= new int[length];
75         fLengths= new int[length];
76         fCount= 0;
77         
78         IScanner scanner= ToolFactory.createScanner(true, true, false, false); // returns comments & whitespace
79 scanner.setSource(fText.toCharArray());
80         int endPos= 0;
81         try {
82             int tokenType;
83             while ((tokenType= scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
84                 int start= scanner.getCurrentTokenStartPosition();
85                 int end= scanner.getCurrentTokenEndPosition()+1;
86                 // Comments are treated as a single token (see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=78063)
87 if (TokenScanner.isComment(tokenType) || tokenType == ITerminalSymbols.TokenNameStringLiteral) {
88                     int dl= fTextTokenComparatorFactory == null ? getCommentStartTokenLength(tokenType) : 0;
89                     recordTokenRange(start, dl);
90                     parseText(start + dl, text.substring(start + dl, end));
91                 } else {
92                     recordTokenRange(start, end - start);
93                 }
94                 endPos= end;
95             }
96         } catch (InvalidInputException ex) {
97             // We couldn't parse part of the input. Fall through and make the rest a single token
98 }
99         // Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=13907
100 if (endPos < length) {
101             recordTokenRange(endPos, length - endPos);
102         }
103     }
104
105     /**
106      * Records the given token range.
107      *
108      * @param start of the token
109      * @param length length of the token
110      * @since 3.3
111      */
112     private void recordTokenRange(int start, int length) {
113         fStarts[fCount]= start;
114         fLengths[fCount]= length;
115         fCount++;
116         if (DEBUG)
117             System.out.println(fText.substring(start, start + length));
118     }
119
120     private void parseText(int start, String JavaDoc text) {
121         ITokenComparator subTokenizer= fTextTokenComparatorFactory == null
122                 ? new JavaTokenComparator(text)
123                 : fTextTokenComparatorFactory.createTokenComparator(text);
124         int count= subTokenizer.getRangeCount();
125         for (int i= 0; i < count; i++) {
126             int subStart= subTokenizer.getTokenStart(i);
127             int subLength= subTokenizer.getTokenLength(i);
128             recordTokenRange(start + subStart, subLength);
129         }
130     }
131
132     /**
133      * Returns the length of the token that
134      * initiates the given comment type.
135      *
136      * @param tokenType
137      * @return the length of the token that start a comment
138      * @since 3.3
139      */
140     private static int getCommentStartTokenLength(int tokenType) {
141         if (tokenType == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
142             return 3;
143         } else if (tokenType == ITerminalSymbols.TokenNameStringLiteral) {
144             return 1;
145         } else {
146             return 2;
147         }
148     }
149
150     /**
151      * Returns the number of tokens in the string.
152      *
153      * @return number of token in the string
154      */
155     public int getRangeCount() {
156         return fCount;
157     }
158
159     /* (non Javadoc)
160      * see ITokenComparator.getTokenStart
161      */
162     public int getTokenStart(int index) {
163         if (index >= 0 && index < fCount)
164             return fStarts[index];
165         if (fCount > 0)
166             return fStarts[fCount-1] + fLengths[fCount-1];
167         return 0;
168     }
169
170     /* (non Javadoc)
171      * see ITokenComparator.getTokenLength
172      */
173     public int getTokenLength(int index) {
174         if (index < fCount)
175             return fLengths[index];
176         return 0;
177     }
178     
179     /**
180      * Returns <code>true</code> if a token given by the first index
181      * matches a token specified by the other <code>IRangeComparator</code> and index.
182      *
183      * @param thisIndex the number of the token within this range comparator
184      * @param other the range comparator to compare this with
185      * @param otherIndex the number of the token within the other comparator
186      * @return <code>true</code> if the token are equal
187      */
188     public boolean rangesEqual(int thisIndex, IRangeComparator other, int otherIndex) {
189         if (other != null && getClass() == other.getClass()) {
190             JavaTokenComparator tc= (JavaTokenComparator) other; // safe cast
191 int thisLen= getTokenLength(thisIndex);
192             int otherLen= tc.getTokenLength(otherIndex);
193             if (thisLen == otherLen)
194                 return fText.regionMatches(false, getTokenStart(thisIndex), tc.fText, tc.getTokenStart(otherIndex), thisLen);
195         }
196         return false;
197     }
198
199     /**
200      * Aborts the comparison if the number of tokens is too large.
201      *
202      * @param length a number on which to base the decision whether to return
203      * <code>true</code> or <code>false</code>
204      * @param maxLength another number on which to base the decision whether to return
205      * <code>true</code> or <code>false</code>
206      * @param other the other <code>IRangeComparator</code> to compare with
207      * @return <code>true</code> to abort a token comparison
208      */
209     public boolean skipRangeComparison(int length, int maxLength, IRangeComparator other) {
210
211         if (getRangeCount() < 50 || other.getRangeCount() < 50)
212             return false;
213
214         if (maxLength < 100)
215             return false;
216
217         if (length < 100)
218             return false;
219
220         if (maxLength > 800)
221             return true;
222
223         if (length < maxLength / 4)
224             return false;
225
226         return true;
227     }
228 }
229
Popular Tags