CharPreprocessorOperation


1   /*
2    * The contents of this file are subject to the terms of the Common Development
3    * and Distribution License (the License). You may not use this file except in
4    * compliance with the License.
5    *
6    * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7    * or http://www.netbeans.org/cddl.txt.
8    *
9    * When distributing Covered Code, include this CDDL Header Notice in each file
10   * and include the License file at http://www.netbeans.org/cddl.txt.
11   * If applicable, add the following below the CDDL Header, with the fields
12   * enclosed by brackets [] replaced by your own identifying information:
13   * "Portions Copyrighted [year] [name of copyright owner]"
14   *
15   * The Original Software is NetBeans. The Initial Developer of the Original
16   * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17   * Microsystems, Inc. All Rights Reserved.
18   */
19  
20  package org.netbeans.lib.lexer;
21  
22  import org.netbeans.lib.editor.util.ArrayUtilities;
23  import org.netbeans.lib.lexer.token.AbstractToken;
24  import org.netbeans.spi.lexer.CharPreprocessor;
25  import org.netbeans.spi.lexer.LexerInput;
26  
27  
28  /**
29   * Character preprocessor delegates all its operation
30   * to this class.
31   * <br/>
32   * Each preprocessor operation has its parent character provider
33   * which is LexerInputOperation if this is there's just one char preprocessor.
34   * <br/>
35   * There can be more preprocessors chained above the LexerInputOperation.
36   * <br/>
37   * The LexerInput operates on top of the top char provider (preprocessor).
38   * @author Miloslav Metelka
39   * @version 1.00
40   */
41  
42  public final class CharPreprocessorOperation implements CharProvider {
43      
44      /** Flag for additional correctness checks (may degrade performance). */
45      private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");
46      
47      /**
48       * Parent char provider from which the characters are read.
49       */
50      private CharProvider parent;
51      
52      /** The actual preprocessor that this operation wraps. */
53      private CharPreprocessor preprocessor;
54      
55      /**
56       * Current reading index among the characters offered by this operation
57       * relative to the current token's begining.
58       */
59      private int readIndex;
60      
61      /**
62       * Max read index that either the client was requesting or which resulted
63       * by calling more than one {@link #outputOriginal(char)}
64       * in the {@link #preprocessChar()}.
65       */
66      private int lookaheadIndex;
67      
68      /**
69       * Read index corresponding to the first char that really had to be preprocessed
70       * for the current token (and is the first one in the prepChars array).
71       */
72      private int prepStartIndex;
73      
74      /**
75       * Read index corresponding to the first non-preprocessed character
76       * after the preprocessed characters.
77       * <br/>
78       * This variable is only valid if (prepStartIndex != lookaheadIndex).
79       */
80      private int prepEndIndex;
81      
82      /**
83       * Characters translated by this preprocessor.
84       * The token's initial characters that do not need any translation
85       * are not stored in this array. Once there is a char resulting
86       * from translation then all the subsequent chars will be stored
87       * in this array even if they did not require translation.
88       * <br/>
89       * Once the token gets created or skipped the indexes are cleared
90       * but the allocated array is retained.
91       */
92      private char[] prepChars = ArrayUtilities.emptyCharArray();
93  
94      /**
95       * Extra read lengths of the input characters corresponding to each preprocessed char
96       * on the output.
97       * <br/>
98       * The length shifts are related to the parent but at the end of the processing
99       * of a particular token they need to retrieve the raw lengths against
100      * the original input text and the extraRawLengthShifts gets populated
101      * if necessary.
102      */
103     private int[] rawLengthShifts;
104     
105     /**
106      * Last character passed to outputOriginal() or outputPreprocessed().
107      */
108     private int lastOutputChar;
109     
110     /** Computed and cached token length. */
111     private int tokenLength;
112     
113     private LexerInputOperation lexerInputOperation;
114     
115     private int tokenEndRawLengthShift;
116 
117     /**
118      * Construct new char preprocessor operation.
119      * @param lexerInputOperation may be null then error notification won't work.
120      */
121     CharPreprocessorOperation(CharProvider parent, CharPreprocessor preprocessor,
122     LexerInputOperation lexerInputOperation) {
123         this.parent = parent;
124         this.preprocessor = preprocessor;
125         this.lexerInputOperation = lexerInputOperation;
126         // Init the preprocessor to use this operation
127         LexerSpiPackageAccessor.get().init(preprocessor, this);
128     }
129     
130     /**
131      * Init the given token if necessary before {@link #tokenApproved()}
132      * gets called.
133      * <br/>
134      * This method is only called on the preprocessor's operation.
135      */
136     public void initApprovedToken(AbstractToken token) {
137         
138     }
139 
140     /**
141      * Read a single character for preprocessing from the underlying input.
142      * <br/>
143      * The character is obtained either from the real input source
144      * or from another (parent) char preprocessor.
145      *
146      * @return valid character or {@link LexerInput#EOF} if there are no more
147      *  characters available on the input.
148      */
149     public int inputRead() {
150         return parent.read();
151     }
152     
153     public void inputBackup(int count) {
154         parent.backup(count);
155     }
156     
157     public void outputOriginal(int ch) {
158         lastOutputChar = ch;
159         if (ch != LexerInput.EOF) {
160             if (prepStartIndex == lookaheadIndex) { // collecting non-preprocessed
161                 prepStartIndex++;
162             } else { // adding to existing prepChars
163                 // leave prepEndIndex as it is now
164             }
165             lookaheadIndex++;
166         }
167     }
168     
169     public void outputPreprocessed(char ch, int extraInputLength) {
170         lastOutputChar = ch;
171         if (prepStartIndex == lookaheadIndex) { // no prepChars yet
172             prepEndIndex = prepStartIndex;
173         } else if (prepEndIndex < lookaheadIndex) {
174             // Add missing output chars
175             do {
176                 addPrepChar(parent.readExisting(prepEndIndex), 0);
177             } while (prepEndIndex < lookaheadIndex);
178         } // else adding to the end of prepChars (last char was prep char)
179         addPrepChar(ch, extraInputLength);
180         lookaheadIndex++; // State that a prep char was added
181     }
182     
183     public int deepRawLength(int length) {
184         return parent.deepRawLength(parentLength(length));
185     }
186     
187     public int deepRawLengthShift(int index) {
188         return rawLengthShift(index) + parent.deepRawLengthShift(index);
189     }
190     
191     private int rawLengthShift(int index) {
192         if (index < prepStartIndex) {
193             return index;
194         } else if (index < prepEndIndex) {
195             return rawLengthShifts[index - prepStartIndex];
196         } else {
197             return totalRawLengthShift();
198         }
199     }
200 
201     /**
202      * Given length here translated into the length in parent.
203      */
204     private int parentLength(int length) {
205         System.out.println("STATUS: length=" + length + ", " + this);
206         if (length > prepStartIndex) {
207             if (length <= prepEndIndex) {
208                 length += rawLengthShifts[length - 1 - prepStartIndex];
209             } else {
210                 length += totalRawLengthShift();
211             }
212         }
213         return length;
214     }
215 
216     private int totalRawLengthShift() {
217         return rawLengthShifts[prepEndIndex - 1 - prepStartIndex];
218     }
219     
220     public void notifyError(String   errorMessage) {
221         if (lexerInputOperation != null) {
222             int parentIndex = parent.readIndex(); // Get the 
223             lexerInputOperation.notifyPreprocessorError(
224                 new CharPreprocessorError(errorMessage, parent.deepRawLength(parentIndex)));
225         }
226     }
227 
228     public int read() {
229         // Check whether any characters need to be preprocessed first
230         if (readIndex == lookaheadIndex) {
231             // Most typical situation - preprocess char
232             LexerSpiPackageAccessor.get().preprocessChar(preprocessor);
233             readIndex++;
234             // Expect only a single char to be put into lastOutputChar
235             if (readIndex == lookaheadIndex) {
236                 return lastOutputChar;
237             } else { // possibly more chars processed or EOF found etc.
238                 readIndex--;
239                 // Check whether EOF was processed (returned)
240                 if (readIndex == lookaheadIndex && lastOutputChar == LexerInput.EOF) {
241                     return LexerInput.EOF;
242                 }
243             }
244             
245         }
246         
247         return readExisting(readIndex++);
248     }
249     
250     public char readExisting(int index) {
251         return (index < prepStartIndex)// below the first preprocessed character
252             ? parent.readExisting(index)
253             : (index < prepEndIndex) // inside prepChars
254                 ? prepChars[index - prepStartIndex]
255                 : parent.readExisting(index + totalRawLengthShift());
256     }
257     
258     public int readIndex() {
259         return readIndex;
260     }
261     
262     public void backup(int count) {
263         readIndex -= count;
264     }
265     
266     public int tokenLength() {
267         return tokenLength;
268     }
269     
270     public void tokenRecognized(int tokenLength) {
271         this.tokenLength = tokenLength;
272         // Modify tokenLength for preprocessed characters
273         parent.tokenRecognized(parentLength(tokenLength));
274     }
275     
276     public PreprocessedTextStorage createPreprocessedTextStorage(CharSequence   rawText,
277     CharProvider.ExtraPreprocessedChars epc) {
278         int pStartIndex;
279         int pEndIndex;
280         int topEndIndex;
281         if (prepStartIndex >= tokenLength) {
282             if (prepEndIndex > tokenLength) {
283                 updateTokenEndRawLengthShift();
284                 pEndIndex = tokenLength - 1;
285                 // Optimize the case when there are lookahead chars
286                 // for the present token and the ending chars could possibly
287                 // be non-preprocessed (prepEndIndex > tokenLength)
288                 while (--pEndIndex >= prepStartIndex && rawLengthShifts[pEndIndex] == tokenEndRawLengthShift) { // not preprocessed
289                 }
290                 pEndIndex += 2;
291             } else // prepEndIndex <= tokenLength
292                 pEndIndex = prepEndIndex;
293             topEndIndex = parentLength(pEndIndex);
294 
295             // Get deep raw lengths
296             for (int i = prepStartIndex; i < pEndIndex; i++) {
297                 rawLengthShifts[i - prepStartIndex] = deepRawLength(i + 1) - (i + 1);
298             }
299             pStartIndex = prepStartIndex;
300 
301         } else { // No preprocessed chars inside token
302             pStartIndex = tokenLength;
303             pEndIndex = tokenLength;
304             topEndIndex = tokenLength;
305         }
306 
307         PreprocessedTextStorage prepStorage;
308         if (epc != null) {
309             parent.collectExtraPreprocessedChars(epc, pStartIndex, pEndIndex, topEndIndex);
310             prepStorage = PreprocessedTextStorage.create(rawText,
311                 prepChars, pEndIndex - pStartIndex, pStartIndex, rawLengthShifts,
312                 epc.extraPrepChars(), epc.extraRawLengthShifts(),
313                 epc.preStartIndex(), epc.postEndIndex());
314             epc.clear();
315             
316         } else { // no extra preprocessed chars
317             prepStorage = PreprocessedTextStorage.create(rawText,
318                 prepChars, pEndIndex - pStartIndex, pStartIndex, rawLengthShifts);
319         }
320         return prepStorage;
321     }
322     
323     private void updateTokenEndRawLengthShift() {
324         tokenEndRawLengthShift = rawLengthShifts[tokenLength - 1 - prepStartIndex];
325     }
326 
327     public void collectExtraPreprocessedChars(ExtraPreprocessedChars epc,
328     int prepStartIndex, int prepEndIndex, int topPrepEndIndex) {
329         if (prepStartIndex < tokenLength) { // Some preprocessed characters
330             // Check for any pre-prepChars
331             int preCount = Math.max(prepStartIndex - this.prepStartIndex, 0);
332             // Check for post-prepChars
333             int postCount;
334             if (this.prepEndIndex > tokenLength) {
335                 updateTokenEndRawLengthShift();
336                 postCount = tokenLength - prepEndIndex;
337                 if (postCount > 0) {
338                     int i = tokenLength - 2;
339                     // Optimize the case when there are lookahead chars
340                     // for the present token and the ending chars could possibly
341                     // be non-preprocessed (prepEndIndex > tokenLength)
342                     while (--i >= prepStartIndex && postCount > 0 && rawLengthShifts[i] == tokenEndRawLengthShift) { // not preprocessed
343                         postCount--;
344                     }
345                 } else // postCount <= 0
346                     postCount = 0;
347 
348             } else { // this.prepEndIndex <= tokenLength
349                 postCount = this.prepEndIndex - prepEndIndex;
350             }
351 
352             assert (preCount >= 0 && postCount >= 0);
353             epc.ensureExtraLength(preCount + postCount);
354             while (--preCount >= 0) {
355                 epc.insert(readExisting(prepStartIndex - 1), deepRawLength(prepStartIndex) - prepStartIndex);
356                 prepStartIndex--;
357             }
358             while (--postCount >= 0) {
359                 epc.append(readExisting(prepEndIndex), deepRawLength(prepEndIndex) - topPrepEndIndex);
360                 prepEndIndex++;
361                 topPrepEndIndex++;
362             }
363         }
364         
365         parent.collectExtraPreprocessedChars(epc, prepStartIndex, prepEndIndex, topPrepEndIndex);
366     }
367     
368     /**
369      * This method is called after the token has been recognized
370      * to clear internal data related to processing of token's characters.
371      */
372     public void tokenApproved() {
373         if (prepStartIndex != lookaheadIndex) { // some prep chars (may be after token length)
374             if (prepStartIndex < tokenLength) { // prep chars before token end
375                 if (prepEndIndex <= tokenLength) { // no preprocessed chars past token end
376                     prepStartIndex = lookaheadIndex; // signal no preprocessed chars
377                 } else { // prepEndIndex > tokenLength => initial prep chars in the next token
378                     // updateTokenLengthParentShift() was already called in this case
379                     for (int i = tokenLength; i < prepEndIndex; i++) {
380                         rawLengthShifts[i] -= tokenEndRawLengthShift;
381                     }
382                     System.arraycopy(prepChars, prepStartIndex, prepChars, 0,
383                             prepEndIndex - prepStartIndex);
384                     System.arraycopy(rawLengthShifts, prepStartIndex, rawLengthShifts, 0,
385                             prepEndIndex - prepStartIndex);
386                     prepStartIndex = 0;
387                     prepEndIndex -= tokenLength;
388                 }
389 
390             } else { // prepStartIndex >= tokenLength
391                 prepStartIndex -= tokenLength;
392                 prepEndIndex -= tokenLength;
393             }
394         } else
395             prepStartIndex -= tokenLength;
396 
397         readIndex -= tokenLength;
398         lookaheadIndex -= tokenLength;
399         parent.tokenApproved();
400 
401         if (testing)
402             consistencyCheck();
403     }
404     
405     /**
406      * Add preprocessed or passed char to prepChars
407      */
408     private void addPrepChar(char ch, int extraInputLength) {
409         int prepCharsLength = prepEndIndex - prepStartIndex;
410         if (prepCharsLength == prepChars.length) { // reallocate
411             prepChars = ArrayUtilities.charArray(prepChars);
412             rawLengthShifts = ArrayUtilities.intArray(rawLengthShifts);
413         }
414         prepChars[prepCharsLength] = ch;
415         int prevRawLengthShift = (prepCharsLength > 0)
416                 ? rawLengthShifts[prepCharsLength -1]
417                 : 0;
418         rawLengthShifts[prepCharsLength] = prevRawLengthShift + extraInputLength;
419         prepEndIndex++;
420     }
421     
422     private void consistencyCheck() {
423         if (readIndex > lookaheadIndex) {
424             throw new IllegalStateException  ("readIndex > lookaheadIndex: " + this);
425         }
426         if (prepStartIndex > lookaheadIndex) {
427             throw new IllegalStateException  ("prepStartIndex > lookaheadIndex: " + this);
428         }
429         if (prepStartIndex != lookaheadIndex && prepStartIndex >= prepEndIndex) {
430             throw new IllegalStateException  ("prepStartIndex >= prepEndIndex: " + this);
431         }
432     }
433     
434     public String   toString() {
435         StringBuilder   sb = new StringBuilder  ();
436         sb.append("readIndex=");
437         sb.append(readIndex);
438         sb.append(", lookaheadIndex=");
439         sb.append(lookaheadIndex);
440         sb.append(", prepStartIndex=");
441         sb.append(prepStartIndex);
442         sb.append(", prepEndIndex=");
443         sb.append(prepEndIndex);
444         return sb.toString();
445     }
446 
447 }
448
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags