KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > netbeans > lib > lexer > batch > SkimLexerInputOperation


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */
19
20 package org.netbeans.lib.lexer.batch;
21
22 import java.io.IOException JavaDoc;
23 import java.io.Reader JavaDoc;
24 import org.netbeans.api.lexer.TokenId;
25 import org.netbeans.lib.lexer.LexerInputOperation;
26 import org.netbeans.lib.lexer.TokenList;
27 import org.netbeans.lib.lexer.token.AbstractToken;
28 import org.netbeans.spi.lexer.LexerInput;
29
30 /**
31  * Lexer input operation over a {@link java.io.Reader}.
32  *
33  * @author Miloslav Metelka
34  * @version 1.00
35  */
36
37 public final class SkimLexerInputOperation<T extends TokenId> extends LexerInputOperation<T> {
38     
39     private static final char[] EMPTY_CHAR_ARRAY = new char[0];
40     
41     /**
42      * Default size for reading char array.
43      */
44     private static final int DEFAULT_READ_CHAR_ARRAY_SIZE = 4096;
45     
46     /**
47      * Minimum size to be read (to have space for reading).
48      */
49     private static final int MIN_READ_SIZE = 512;
50     
51     private static final int DEFAULT_CLUSTER_SIZE = 4096;
52     
53     /**
54      * Maximum fragmentation factor for token character arrays.
55      * <br>
56      * If there is not enough space in the tokenCharArray
57      * to copy a token's characters there then if the token's length
58      * will be greater than this threshold then the token will get
59      * an extra character buffer just for itself and there will
60      * still be chance to use the present tokenCharArray for tokens
61      * with lower length.
62      */
63     private static final int MAX_UNUSED_CLUSTER_SIZE_FRACTION = 50;
64     
65
66     /**
67      * Reader as a primary source of characters that are further
68      * copied and cached.
69      */
70     private Reader JavaDoc reader;
71     
72     /**
73      * Array holding the read characters.
74      */
75     private char[] readCharArray;
76     
77     /**
78      * Character sequence holding the characters to be read.
79      */
80     private CharSequence JavaDoc readCharSequence;
81     
82     /**
83      * Index of a first character in the token being currently recognized.
84      */
85     private int readStartIndex;
86     
87     /**
88      * End of valid chars in readCharArray (points to first invalid char).
89      */
90     private int readEndIndex;
91     
92     /**
93      * Whether EOF was read from reader already or not.
94      */
95     private boolean eofRead;
96     
97     /**
98      * Actual token cluster where the tokens are being placed.
99      */
100     private SkimTokenList<T> cluster;
101
102     private int clusterTextEndIndex;
103     
104     private int defaultClusterSize = DEFAULT_CLUSTER_SIZE;
105     
106     /**
107      * Starting offset of the cluster currently being used.
108      */
109     private int clusterStartOffset;
110     
111     /**
112      * How much the offset is ahead of the token's text offset
113      * in the cluster. The tokens that get skipped and flyweight tokens
114      * increase this value because their text is not physically copied
115      * into the clusters character data but they increase the offset.
116      */
117     private int offsetShift;
118     
119     public SkimLexerInputOperation(TokenList<T> tokenList, Reader JavaDoc reader) {
120         super(tokenList, 0, null);
121         this.reader = reader;
122         this.readCharArray = new char[DEFAULT_READ_CHAR_ARRAY_SIZE];
123     }
124     
125     public SkimLexerInputOperation(TokenList<T> tokenList, CharSequence JavaDoc readCharSequence) {
126         super(tokenList, 0, null);
127         this.readCharSequence = readCharSequence;
128         this.readEndIndex = readCharSequence.length();
129     }
130     
131     public int read(int index) { // index >= 0 is guaranteed by contract
132 index += readStartIndex;
133         if (index < readEndIndex) {
134             return (readCharArray != null)
135                 ? readCharArray[index]
136                 : readCharSequence.charAt(index);
137
138         } else { // must read next or return EOF
139 if (!eofRead) {
140                 eofRead = (readCharArray != null)
141                     ? readNextCharArray()
142                     : true; // using readCharSequence -> no more chars
143
144                 return read(index);
145
146             } else {
147                 return LexerInput.EOF;
148             }
149         }
150     }
151     
152     public char readExisting(int index) {
153         return (readCharArray != null)
154             ? readCharArray[index]
155             : readCharSequence.charAt(index);
156     }
157     
158     public void approveToken(AbstractToken<T> token) {
159         int tokenLength = token.length();
160         if (isSkipToken(token)) {
161             preventFlyToken();
162             skipChars(tokenLength());
163             
164         } else if (token.isFlyweight()) {
165             assert isFlyTokenAllowed();
166             flyTokenAdded();
167             skipChars(tokenLength);
168
169         } else { // non-flyweight token => must be L0Token instance
170 if (clusterTextEndIndex != 0) { // valid cluster exists
171 // Check whether token fits into cluster's char array
172 if (tokenLength + clusterTextEndIndex > cluster.getText().length) {
173                     // Cannot fit the token's text into current cluster
174 finishCluster();
175                 }
176             }
177
178             if (clusterTextEndIndex == 0) { // allocate new cluster
179 int clusterSize = defaultClusterSize;
180                 if (clusterSize < tokenLength) { // cluster just for one token
181 clusterSize = tokenLength;
182                 }
183                 defaultClusterSize = clusterSize;
184                 cluster = new SkimTokenList<T>((CopyTextTokenList<T>)tokenList(),
185                         clusterStartOffset, new char[clusterSize]);
186             }
187
188             // Now it's clear that the token will fit into the cluster's text
189 // TODO for DirectCharSequence use more efficient way
190 char[] clusterText = cluster.getText();
191             if (readCharArray != null) {
192                 System.arraycopy(readCharArray, readStartIndex, clusterText,
193                         clusterTextEndIndex, tokenLength);
194             } else { // using readCharSequence
195 for (int i = 0; i < tokenLength; i++) {
196                     clusterText[clusterTextEndIndex + i]
197                             = readCharSequence.charAt(readStartIndex + i);
198                 }
199             }
200             
201             int rawOffset = (offsetShift << 16) | clusterTextEndIndex;
202             token.setTokenList(cluster);
203             token.setRawOffset(rawOffset);
204             clusterTextEndIndex += tokenLength;
205             clearFlySequence();
206         }
207
208         readStartIndex += tokenLength;
209         tokenApproved();
210     }
211
212     private void skipChars(int skipLength) {
213         if (clusterTextEndIndex != 0) { // cluster already populated
214 if (offsetShift + skipLength > Short.MAX_VALUE) {
215                 // Cannot advance offset shift without overflowing -> cluster is finished
216 finishCluster();
217                 clusterStartOffset += skipLength;
218
219             } else { // relOffset will fit into current cluster
220 offsetShift += skipLength;
221             }
222
223         } else { // cluster is null -> can shift cluster's start offset
224 clusterStartOffset += skipLength;
225         }
226     }
227     
228     public void finish() {
229         if (clusterTextEndIndex != 0) {
230             finishCluster();
231         }
232     }
233
234     private void finishCluster() {
235         // If there would be too much unused space in the cluster's char array
236 // then it will be reallocated.
237 int clusterTextLength = cluster.getText().length;
238         if (clusterTextLength / MAX_UNUSED_CLUSTER_SIZE_FRACTION
239                 > (clusterTextLength - clusterTextEndIndex)
240         ) { // Fragmentation -> reallocate cluster's char array
241 char[] newText = new char[clusterTextEndIndex];
242             System.arraycopy(cluster.getText(), 0, newText, 0, clusterTextEndIndex);
243             cluster.setText(newText);
244         }
245         clusterStartOffset += clusterTextEndIndex + offsetShift;
246         clusterTextEndIndex = 0;
247         offsetShift = 0;
248         cluster = null; // cluster no longer valid
249 }
250     
251     private boolean readNextCharArray() {
252         // Copy everything from present readStartIndex till readEndIndex
253 int retainLength = readEndIndex - readStartIndex;
254         int minReadSize = readCharArray.length - retainLength;
255         char[] newReadCharArray = readCharArray; // by default take original one
256 if (minReadSize < MIN_READ_SIZE) { // allocate new
257 // double the current array's size
258 newReadCharArray = new char[readCharArray.length * 2];
259         }
260         System.arraycopy(readCharArray, readStartIndex, newReadCharArray, 0, retainLength);
261         readCharArray = newReadCharArray;
262         readStartIndex = 0;
263         readEndIndex = retainLength;
264         
265         boolean eof = false;
266         while (readEndIndex < readCharArray.length) {
267             int readSize;
268             try {
269                 readSize = reader.read(readCharArray, readEndIndex,
270                     readCharArray.length - readEndIndex);
271             } catch (IOException JavaDoc e) {
272                 // The exception is silently ignored here
273 // This should generally not happen - a wrapping reader
274 // should be used that will catch and process the IO exceptions.
275 readSize = -1;
276             }
277             if (readSize == -1) {
278                 eof = true;
279                 try {
280                     reader.close();
281                 } catch (IOException JavaDoc e) {
282                     // The exception is silently ignored here
283 // This should generally not happen - a wrapping reader
284 // should be used that will catch and process the IO exceptions.
285 }
286                 break;
287             } else {
288                 readEndIndex += readSize;
289             }
290         }
291         return eof;
292     }
293
294 }
295
Popular Tags