PreprocessedTextStorage


1   /*
2    * The contents of this file are subject to the terms of the Common Development
3    * and Distribution License (the License). You may not use this file except in
4    * compliance with the License.
5    *
6    * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7    * or http://www.netbeans.org/cddl.txt.
8    *
9    * When distributing Covered Code, include this CDDL Header Notice in each file
10   * and include the License file at http://www.netbeans.org/cddl.txt.
11   * If applicable, add the following below the CDDL Header, with the fields
12   * enclosed by brackets [] replaced by your own identifying information:
13   * "Portions Copyrighted [year] [name of copyright owner]"
14   *
15   * The Original Software is NetBeans. The Initial Developer of the Original
16   * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17   * Microsystems, Inc. All Rights Reserved.
18   */
19  
20  package org.netbeans.lib.lexer;
21  
22  import org.netbeans.lib.editor.util.ArrayUtilities;
23  import org.netbeans.lib.editor.util.CharSequenceUtilities;
24  import org.netbeans.lib.editor.util.CharSubSequence;
25  
26  /**
27   * Storage of the preprocessed text specific for batch and mutable setups.
28   * <br/>
29   * For mutable setups the extra length shifts need to be stored.
30   * <br/>
31   * For storage costs cutting only a maximum lookahead is stored (not individual
32   * lookaheads).
33   *
34   * @author Miloslav Metelka
35   * @version 1.00
36   */
37  
38  public abstract class PreprocessedTextStorage implements CharSequence   {
39      
40      /** Flag for additional correctness checks (may degrade performance). */
41      private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");
42      
43      /**
44       * Create preprocessed characters storage.
45       * 
46       * @param rawText non-null raw text before preprocessing.
47       * @param preprocessedChars non-null array of preprocessed characters
48       * @param preprocessedCharsLength >=0 number of valid characters in the preprocessedChars array.
49       * @param preprocessedStartIndex index of first preprocessed character in the raw text.
50       * @param preprocessedRawLengthShifts non-null array of shifts of the particular
51       *  end of a character in the raw text against the preprocessed text.
52       *  For example "abc" will produce preprocessed "abc"
53       *  and the raw length shifts would be { 1-1=0, 7-2=5, 8-3=5 }.
54       * 
55       */
56      public static PreprocessedTextStorage create(CharSequence   rawText,
57      char[] preprocessedChars, int preprocessedCharsLength,
58      int preprocessedStartIndex, int[] preprocessedRawLengthShifts) {
59          char[] preprocessedCharsCopy = ArrayUtilities.charArray(preprocessedChars, preprocessedCharsLength);
60          // Get maximum raw length shift (the one of the last character of preprocessedChars)
61          int totalRawLengthShift = preprocessedRawLengthShifts[preprocessedCharsLength - 1];
62          // Create appropriate storage according to the max. rawLengthShift
63          PreprocessedTextStorage storage;
64          if (totalRawLengthShift <= Byte.MAX_VALUE) { // Use bytes
65              byte[] arr = new byte[preprocessedCharsLength];
66              for (int i = preprocessedCharsLength - 1; i >= 0; i--)
67                  arr[i] = (byte)preprocessedRawLengthShifts[i];
68              storage = new ByteRawIndex(rawText, preprocessedCharsCopy, preprocessedStartIndex,
69                      preprocessedCharsLength, totalRawLengthShift, arr);
70              
71          } else if (totalRawLengthShift <= Short.MAX_VALUE) { // Use shorts
72              short[] arr = new short[preprocessedCharsLength];
73              for (int i = preprocessedCharsLength - 1; i >= 0; i--)
74                  arr[i] = (short)preprocessedRawLengthShifts[i];
75              storage = new ShortRawIndex(rawText, preprocessedCharsCopy, preprocessedStartIndex,
76                      preprocessedCharsLength, totalRawLengthShift, arr);
77              
78          } else { // Use ints
79              int[] arr = new int[preprocessedCharsLength];
80              System.arraycopy(preprocessedChars, 0, arr, 0, preprocessedCharsLength);
81              storage = new IntRawIndex(rawText, preprocessedCharsCopy, preprocessedStartIndex,
82                      preprocessedCharsLength, totalRawLengthShift, arr);
83          }
84  
85          if (testing)
86              storage.consistencyCheck();
87          return storage;
88      }
89  
90      /**
91       * Create preprocessed characters storage.
92       * 
93       * @param rawText non-null raw text before preprocessing.
94       * @param preprocessedChars non-null array of preprocessed characters
95       * @param preprocessedCharsLength >=0 number of valid characters in the preprocessedChars array.
96       * @param preprocessedStartIndex index of first preprocessed character in the raw text.
97       * @param preprocessedRawLengthShifts non-null array of shifts of the particular
98       *  end of a character in the raw text against the preprocessed text.
99       *  For example "abc" will produce preprocessed "abc"
100      *  and the raw length shifts would be { 1-1=0, 7-2=5, 8-3=5 }.
101      * 
102      */
103     public static PreprocessedTextStorage create(CharSequence   rawText, char[] preprocessedChars, int preprocessedCharsLength,
104     int preprocessedStartIndex, int[] preprocessedRawLengthShifts,
105     char[] extraPreprocessedChars, int[] extraRawLengthShifts, int preStartIndex, int postEndIndex) {
106         int extraPreCharsLength = (extraPreprocessedChars.length - preStartIndex);
107         preprocessedStartIndex -= extraPreCharsLength;
108         int length = extraPreCharsLength + preprocessedCharsLength + postEndIndex;
109         
110         // Create copy of the characters
111         char[] preprocessedCharsCopy = new char[length];
112         System.arraycopy(extraPreprocessedChars, preStartIndex, preprocessedCharsCopy, 0, extraPreCharsLength);
113         System.arraycopy(preprocessedChars, 0, preprocessedCharsCopy, extraPreCharsLength, preprocessedCharsLength);
114         System.arraycopy(extraPreprocessedChars, 0, preprocessedCharsCopy, extraPreCharsLength + preprocessedCharsLength, postEndIndex);
115 
116         // Get maximum raw length shift (the one of the last character of preprocessedChars)
117         int totalRawLengthShift = (postEndIndex > 0)
118                 ? extraRawLengthShifts[postEndIndex - 1]
119                 : (preprocessedCharsLength > 0)
120                     ? preprocessedRawLengthShifts[preprocessedCharsLength - 1]
121                     // Otherwise get last of pre-chars
122                     : extraRawLengthShifts[extraPreprocessedChars.length - 1];
123         
124 
125         
126         // Create appropriate storage according to the max. rawLengthShift
127         int ind = length - 1;
128         PreprocessedTextStorage storage;
129         if (totalRawLengthShift <= Byte.MAX_VALUE) { // Use bytes
130             byte[] arr = new byte[length];
131             for (int i = postEndIndex - 1; i >= 0; i--)
132                 arr[ind--] = (byte)extraRawLengthShifts[i];
133             for (int i = preprocessedCharsLength - 1; i >= 0; i--)
134                 arr[ind--] = (byte)preprocessedRawLengthShifts[i];
135             for (int i = extraPreprocessedChars.length - 1; i >= preStartIndex; i--)
136                 arr[ind--] = (byte)extraRawLengthShifts[i];
137             storage = new ByteRawIndex(rawText, preprocessedCharsCopy, preprocessedStartIndex, length, totalRawLengthShift, arr);
138             
139         } else if (totalRawLengthShift <= Short.MAX_VALUE) { // Use shorts
140             short[] arr = new short[length];
141             for (int i = postEndIndex - 1; i >= 0; i--)
142                 arr[ind--] = (short)extraRawLengthShifts[i];
143             for (int i = preprocessedCharsLength - 1; i >= 0; i--)
144                 arr[ind--] = (short)preprocessedRawLengthShifts[i];
145             for (int i = extraPreprocessedChars.length - 1; i >= preStartIndex; i--)
146                 arr[ind--] = (short)extraRawLengthShifts[i];
147             storage = new ShortRawIndex(rawText, preprocessedCharsCopy, preprocessedStartIndex, length, totalRawLengthShift, arr);
148             
149         } else { // Use ints
150             // System.arraycopy() could be used but usually bytes (or shorts for long tokens)
151             // would be used anyway so to eliminate errors use the copy-pasted code from above
152             int[] arr = new int[length];
153             for (int i = postEndIndex - 1; i >= 0; i--)
154                 arr[ind--] = extraRawLengthShifts[i];
155             for (int i = preprocessedCharsLength - 1; i >= 0; i--)
156                 arr[ind--] = preprocessedRawLengthShifts[i];
157             for (int i = extraPreprocessedChars.length - 1; i >= preStartIndex; i--)
158                 arr[ind--] = extraRawLengthShifts[i];
159             storage = new IntRawIndex(rawText, preprocessedCharsCopy, preprocessedStartIndex, length, totalRawLengthShift, arr);
160         }
161         
162         if (testing)
163             storage.consistencyCheck();
164         return storage;
165     }
166 
167 
168     /**
169      * Reference to original non-preprocessed text.
170      */
171     private final CharSequence   rawText; // 12 bytes (8-Object + 4)
172     
173     /**
174      * Preprocessed characters.
175      */
176     private final char[] preprocessedChars; // 16 bytes
177 
178     /**
179      * Index of the first preprocessed char in the rawText.
180      */
181     private final int preprocessedStartIndex; // 20 bytes
182     
183     /**
184      * Raw index shift of the first char after preprocessedChars characters.
185      */
186     private final int totalRawLengthShift; // 24 bytes
187     
188     /**
189      * Cache the length of this text storage. Although it could be computed
190      * dynamically by using rawText.length() this avoids nested
191      * length() calls.
192      */
193     private final int length; // 28 bytes
194     
195     protected PreprocessedTextStorage(CharSequence   rawText, char[] preprocessedChars,
196     int preprocessedStartIndex, int length, int totalRawLengthShift) {
197         this.rawText = rawText;
198         this.preprocessedChars = preprocessedChars;
199         this.preprocessedStartIndex = preprocessedStartIndex;
200         // Compute end raw index without delegating to rawText methods e.g. length()
201         this.totalRawLengthShift = totalRawLengthShift;
202         this.length = length;
203     }
204     
205     protected abstract int prepRawLengthShift(int index);
206     
207     /**
208      * Raw length for the given length.
209      *
210      * @param length length in this character sequence.
211      * @return raw length for the given length.
212      */
213     public final int rawLength(int length) {
214         if (length > preprocessedStartIndex) {
215             int prepLength = length - preprocessedStartIndex;
216             if (prepLength <= preprocessedChars.length) {
217                 length += prepRawLengthShift(prepLength - 1);
218             } else {
219                 length += totalRawLengthShift;
220             }
221         }
222         return length;
223     }
224     
225     /**
226      * Raw length shift corresponding to the given index.
227      *
228      * @param index >=0 and <length() index in this text storage.
229      * @return raw length shift corresponding to the given index.
230      */
231     public final int rawLengthShift(int index) {
232         if (index < preprocessedStartIndex) {
233             return index;
234         } else {
235             index -= preprocessedStartIndex;
236             if (index <= preprocessedChars.length) {
237                 return prepRawLengthShift(index) ;
238             } else { // past the end of preprocessed characters
239                 return totalRawLengthShift;
240             }
241         }
242     }
243     
244     public final char charAt(int index) {
245         CharSequenceUtilities.checkIndexValid(index, length);
246         if (index < preprocessedStartIndex) {
247             return rawText.charAt(index);
248         } else {
249             int prepIndex = index - preprocessedStartIndex;
250             if (prepIndex < preprocessedChars.length) {
251                 return preprocessedChars[prepIndex];
252             } else { // past the end of preprocessed characters
253                 return rawText.charAt(index + totalRawLengthShift);
254             }
255         }
256     }
257 
258     public final CharSequence   subSequence(int start, int end) {
259         return new CharSubSequence(this, start, end);
260     }
261 
262     public final int length() {
263         return length;
264     }
265     
266     private void consistencyCheck() {
267         // Check that raw length shifts have increasing order
268         int lastRLS = 0;
269         for (int i = 0; i < preprocessedChars.length; i++) {
270             int rls = prepRawLengthShift(i);
271             if (rls < lastRLS) {
272                 throw new IllegalStateException  ("rls=" + rls // NOI18N
273                         + " < lastRLS=" + lastRLS + " at index=" + i); // NOI18N
274             }
275             lastRLS = rls;
276         }
277     }
278 
279     private static final class ByteRawIndex extends PreprocessedTextStorage {
280         
281         private final byte[] preprocessedRawLengthShifts; // 24 bytes
282         
283         ByteRawIndex(CharSequence   rawText, char[] preprocessedChars, int preprocessedStartIndex,
284         int length, int totalRawLengthShift, byte[] preprocessedRawLengthShifts) {
285             super(rawText, preprocessedChars, preprocessedStartIndex, length, totalRawLengthShift);
286             this.preprocessedRawLengthShifts = preprocessedRawLengthShifts;
287         }
288 
289         protected final int prepRawLengthShift(int index) {
290             return preprocessedRawLengthShifts[index];
291         }
292 
293     }
294     
295     private static final class ShortRawIndex extends PreprocessedTextStorage {
296         
297         private final short[] preprocessedRawLengthShifts; // 24 bytes
298 
299         ShortRawIndex(CharSequence   rawText, char[] preprocessedChars, int preprocessedStartIndex,
300         int length, int totalRawLengthShift, short[] preprocessedRawLengthShifts) {
301             super(rawText, preprocessedChars, preprocessedStartIndex, length, totalRawLengthShift);
302             this.preprocessedRawLengthShifts = preprocessedRawLengthShifts;
303         }
304 
305         protected final int prepRawLengthShift(int index) {
306             return preprocessedRawLengthShifts[index];
307         }
308 
309     }
310 
311     private static final class IntRawIndex extends PreprocessedTextStorage {
312         
313         private final int[] preprocessedRawLengthShifts; // 24 bytes
314 
315         IntRawIndex(CharSequence   rawText, char[] preprocessedChars, int preprocessedStartIndex,
316         int length, int totalRawLengthShift, int[] preprocessedRawLengthShifts) {
317             super(rawText, preprocessedChars, preprocessedStartIndex, length, totalRawLengthShift);
318             this.preprocessedRawLengthShifts = preprocessedRawLengthShifts;
319         }
320 
321         protected final int prepRawLengthShift(int index) {
322             return preprocessedRawLengthShifts[index];
323         }
324 
325     }
326     
327 }
328
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags