PublicScanner


1   /*******************************************************************************
2    * Copyright (c) 2000, 2007 IBM Corporation and others.
3    * All rights reserved. This program and the accompanying materials
4    * are made available under the terms of the Eclipse Public License v1.0
5    * which accompanies this distribution, and is available at
6    * http://www.eclipse.org/legal/epl-v10.html
7    *
8    * Contributors:
9    *     IBM Corporation - initial API and implementation
10   *******************************************************************************/
11  package org.eclipse.jdt.internal.core.util;
12  
13  import org.eclipse.jdt.core.compiler.CharOperation;
14  import org.eclipse.jdt.core.compiler.IScanner;
15  import org.eclipse.jdt.core.compiler.ITerminalSymbols;
16  import org.eclipse.jdt.core.compiler.InvalidInputException;
17  import org.eclipse.jdt.internal.compiler.CompilationResult;
18  import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants;
19  import org.eclipse.jdt.internal.compiler.parser.NLSTag;
20  import org.eclipse.jdt.internal.compiler.parser.ScannerHelper;
21  import org.eclipse.jdt.internal.compiler.util.Util;
22  
23  public class PublicScanner implements IScanner, ITerminalSymbols {
24      
25      //public int newIdentCount = 0;
26      
27      /* APIs ares
28       - getNextToken() which return the current type of the token
29         (this value is not memorized by the scanner)
30       - getCurrentTokenSource() which provides with the token "REAL" source
31         (aka all unicode have been transformed into a correct char)
32       - sourceStart gives the position into the stream
33       - currentPosition-1 gives the sourceEnd position into the stream 
34      */
35      public long sourceLevel;
36      public long complianceLevel;
37  
38      // 1.4 feature 
39      public boolean useAssertAsAnIndentifier = false;
40      //flag indicating if processed source contains occurrences of keyword assert 
41      public boolean containsAssertKeyword = false; 
42      
43      // 1.5 feature
44      public boolean useEnumAsAnIndentifier = false;
45      
46      public boolean recordLineSeparator = false;
47      public char currentCharacter;
48      public int startPosition;
49      public int currentPosition;
50      public int initialPosition, eofPosition;
51      // after this position eof are generated instead of real token from the source
52  
53      public boolean tokenizeComments = false;
54      public boolean tokenizeWhiteSpace = false;
55  
56      //source should be viewed as a window (aka a part)
57      //of a entire very large stream
58      public char source[];
59  
60      //unicode support
61      public char[] withoutUnicodeBuffer;
62      public int withoutUnicodePtr; //when == 0 ==> no unicode in the current token
63      public boolean unicodeAsBackSlash = false;
64  
65      public boolean scanningFloatLiteral = false;
66  
67      //support for /** comments
68      public final static int COMMENT_ARRAYS_SIZE = 30;
69      public int[] commentStops = new int[COMMENT_ARRAYS_SIZE];
70      public int[] commentStarts = new int[COMMENT_ARRAYS_SIZE];
71      public int[] commentTagStarts = new int[COMMENT_ARRAYS_SIZE];
72      public int commentPtr = -1; // no comment test with commentPtr value -1
73      protected int lastCommentLinePosition = -1;
74      
75      // task tag support
76      public char[][] foundTaskTags = null;
77      public char[][] foundTaskMessages;
78      public char[][] foundTaskPriorities = null;
79      public int[][] foundTaskPositions;
80      public int foundTaskCount = 0;
81      public char[][] taskTags = null;
82      public char[][] taskPriorities = null;
83      public boolean isTaskCaseSensitive = true;
84      
85      //diet parsing support - jump over some method body when requested
86      public boolean diet = false;
87  
88      //support for the  poor-line-debuggers ....
89      //remember the position of the cr/lf
90      public int[] lineEnds = new int[250];
91      public int linePtr = -1;
92      public boolean wasAcr = false;
93  
94      public static final String   END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$
95  
96      public static final String   INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$
97      public static final String   INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$
98      public static final String   INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant";  //$NON-NLS-1$
99      public static final String   INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$
100     public static final String   INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$
101     public static final String   INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$
102     public static final String   INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$
103     public static final String   INVALID_LOW_SURROGATE = "Invalid_Low_Surrogate"; //$NON-NLS-1$
104     public static final String   INVALID_HIGH_SURROGATE = "Invalid_High_Surrogate"; //$NON-NLS-1$
105 
106     public static final String   NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$
107     public static final String   UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$
108     public static final String   UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$
109     public static final String   INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$
110     public static final String   INVALID_DIGIT = "Invalid_Digit"; //$NON-NLS-1$
111     private static final int[] EMPTY_LINE_ENDS = Util.EMPTY_INT_ARRAY;
112 
113     //----------------optimized identifier managment------------------
114     static final char[] charArray_a = new char[] {'a'}, 
115         charArray_b = new char[] {'b'}, 
116         charArray_c = new char[] {'c'}, 
117         charArray_d = new char[] {'d'}, 
118         charArray_e = new char[] {'e'}, 
119         charArray_f = new char[] {'f'}, 
120         charArray_g = new char[] {'g'}, 
121         charArray_h = new char[] {'h'}, 
122         charArray_i = new char[] {'i'}, 
123         charArray_j = new char[] {'j'}, 
124         charArray_k = new char[] {'k'}, 
125         charArray_l = new char[] {'l'}, 
126         charArray_m = new char[] {'m'}, 
127         charArray_n = new char[] {'n'}, 
128         charArray_o = new char[] {'o'}, 
129         charArray_p = new char[] {'p'}, 
130         charArray_q = new char[] {'q'}, 
131         charArray_r = new char[] {'r'}, 
132         charArray_s = new char[] {'s'}, 
133         charArray_t = new char[] {'t'}, 
134         charArray_u = new char[] {'u'}, 
135         charArray_v = new char[] {'v'}, 
136         charArray_w = new char[] {'w'}, 
137         charArray_x = new char[] {'x'}, 
138         charArray_y = new char[] {'y'}, 
139         charArray_z = new char[] {'z'}; 
140 
141     static final char[] initCharArray = 
142         new char[] {'\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000'}; 
143     static final int TableSize = 30, InternalTableSize = 6; //30*6 =210 entries
144     
145     public static final int OptimizedLength = 7;
146     public /*static*/ final char[][][][] charArray_length = 
147         new char[OptimizedLength][TableSize][InternalTableSize][]; 
148     // support for detecting non-externalized string literals
149     public static final char[] TAG_PREFIX= "//$NON-NLS-".toCharArray(); //$NON-NLS-1$
150     public static final int TAG_PREFIX_LENGTH= TAG_PREFIX.length;
151     public static final char TAG_POSTFIX= '$';
152     public static final int TAG_POSTFIX_LENGTH= 1;
153     private NLSTag[] nlsTags = null;
154     protected int nlsTagsPtr;
155     public boolean checkNonExternalizedStringLiterals;
156     
157     // generic support
158     public boolean returnOnlyGreater = false;
159     
160     /*static*/ {
161         for (int i = 0; i < 6; i++) {
162             for (int j = 0; j < TableSize; j++) {
163                 for (int k = 0; k < InternalTableSize; k++) {
164                     this.charArray_length[i][j][k] = initCharArray;
165                 }
166             }
167         }
168     }
169     /*static*/ int newEntry2 = 0, 
170         newEntry3 = 0, 
171         newEntry4 = 0, 
172         newEntry5 = 0, 
173         newEntry6 = 0;
174     public boolean insideRecovery = false;
175 
176     public static final int RoundBracket = 0;
177     public static final int SquareBracket = 1;
178     public static final int CurlyBracket = 2;   
179     public static final int BracketKinds = 3;
180     
181     // extended unicode support
182     public static final int LOW_SURROGATE_MIN_VALUE = 0xDC00;
183     public static final int HIGH_SURROGATE_MIN_VALUE = 0xD800;
184     public static final int HIGH_SURROGATE_MAX_VALUE = 0xDBFF;
185     public static final int LOW_SURROGATE_MAX_VALUE = 0xDFFF;
186 
187 public PublicScanner() {
188     this(false /*comment*/, false /*whitespace*/, false /*nls*/, ClassFileConstants.JDK1_3 /*sourceLevel*/, null/*taskTag*/, null/*taskPriorities*/, true /*taskCaseSensitive*/);
189 }
190 
191 public PublicScanner(
192         boolean tokenizeComments, 
193         boolean tokenizeWhiteSpace, 
194         boolean checkNonExternalizedStringLiterals, 
195         long sourceLevel,
196         long complianceLevel,
197         char[][] taskTags,
198         char[][] taskPriorities,
199         boolean isTaskCaseSensitive) {
200 
201     this.eofPosition = Integer.MAX_VALUE;
202     this.tokenizeComments = tokenizeComments;
203     this.tokenizeWhiteSpace = tokenizeWhiteSpace;
204     this.sourceLevel = sourceLevel;
205     this.complianceLevel = complianceLevel;
206     this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
207     this.taskTags = taskTags;
208     this.taskPriorities = taskPriorities;
209     this.isTaskCaseSensitive = isTaskCaseSensitive;
210 }
211 
212 public PublicScanner(
213         boolean tokenizeComments, 
214         boolean tokenizeWhiteSpace, 
215         boolean checkNonExternalizedStringLiterals, 
216         long sourceLevel,
217         char[][] taskTags,
218         char[][] taskPriorities,
219         boolean isTaskCaseSensitive) {
220 
221     this(
222         tokenizeComments,
223         tokenizeWhiteSpace,
224         checkNonExternalizedStringLiterals,
225         sourceLevel,
226         sourceLevel,
227         taskTags,
228         taskPriorities,
229         isTaskCaseSensitive);
230 }
231 
232 public final boolean atEnd() {
233     // This code is not relevant if source is 
234     // Only a part of the real stream input
235 
236     return this.eofPosition <= this.currentPosition;
237 }
238 
239 // chech presence of task: tags
240 // TODO (frederic) see if we need to take unicode characters into account...
241 public void checkTaskTag(int commentStart, int commentEnd) throws InvalidInputException {
242     char[] src = this.source;
243     
244     // only look for newer task: tags
245     if (this.foundTaskCount > 0
246         && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
247         return;
248     }
249     int foundTaskIndex = this.foundTaskCount;
250     char previous = src[commentStart+1]; // should be '*' or '/'
251     for (
252         int i = commentStart + 2; i < commentEnd && i < this.eofPosition; i++) {
253         char[] tag = null;
254         char[] priority = null;
255         // check for tag occurrence only if not ambiguous with javadoc tag
256         if (previous != '@') {
257             nextTag : for (int itag = 0; itag < this.taskTags.length; itag++) {
258                 tag = this.taskTags[itag];
259                 int tagLength = tag.length;
260                 if (tagLength == 0) continue nextTag;
261     
262                 // ensure tag is not leaded with letter if tag starts with a letter
263                 if (ScannerHelper.isJavaIdentifierStart(tag[0])) {
264                     if (ScannerHelper.isJavaIdentifierPart(previous)) {
265                         continue nextTag;
266                     }
267                 }
268     
269                 for (int t = 0; t < tagLength; t++) {
270                     char sc, tc;
271                     int x = i+t;
272                     if (x >= this.eofPosition || x >= commentEnd) continue nextTag;
273                     if ((sc = src[i + t]) != (tc = tag[t])) {                                                                                   // case sensitive check
274                         if (this.isTaskCaseSensitive || (ScannerHelper.toLowerCase(sc) != ScannerHelper.toLowerCase(tc))) {     // case insensitive check
275                             continue nextTag;
276                         }
277                     }
278                 }
279                 // ensure tag is not followed with letter if tag finishes with a letter
280                 if (i+tagLength < commentEnd && ScannerHelper.isJavaIdentifierPart(src[i+tagLength-1])) {
281                     if (ScannerHelper.isJavaIdentifierPart(src[i + tagLength]))
282                         continue nextTag;
283                 }
284                 if (this.foundTaskTags == null) {
285                     this.foundTaskTags = new char[5][];
286                     this.foundTaskMessages = new char[5][];
287                     this.foundTaskPriorities = new char[5][];
288                     this.foundTaskPositions = new int[5][];
289                 } else if (this.foundTaskCount == this.foundTaskTags.length) {
290                     System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
291                     System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
292                     System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
293                     System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0, this.foundTaskCount);
294                 }
295                 
296                 priority = this.taskPriorities != null && itag < this.taskPriorities.length
297                             ? this.taskPriorities[itag]
298                             : null;
299                 
300                 this.foundTaskTags[this.foundTaskCount] = tag;
301                 this.foundTaskPriorities[this.foundTaskCount] = priority;
302                 this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
303                 this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
304                 this.foundTaskCount++;
305                 i += tagLength - 1; // will be incremented when looping
306                 break nextTag;
307             }
308         }
309         previous = src[i];
310     }
311     boolean containsEmptyTask = false;
312     for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
313         // retrieve message start and end positions
314         int msgStart = this.foundTaskPositions[i][0] + this.foundTaskTags[i].length;
315         int max_value = i + 1 < this.foundTaskCount
316                 ? this.foundTaskPositions[i + 1][0] - 1
317                 : commentEnd - 1;
318         // at most beginning of next task
319         if (max_value < msgStart) {
320             max_value = msgStart; // would only occur if tag is before EOF.
321         }
322         int end = -1;
323         char c;
324         for (int j = msgStart; j < max_value; j++) {
325             if ((c = src[j]) == '\n' || c == '\r') {
326                 end = j - 1;
327                 break;
328             }
329         }
330         if (end == -1) {
331             for (int j = max_value; j > msgStart; j--) {
332                 if ((c = src[j]) == '*') {
333                     end = j - 1;
334                     break;
335                 }
336             }
337             if (end == -1)
338                 end = max_value;
339         }
340         if (msgStart == end) {
341             // if the description is empty, we might want to see if two tags are not sharing the same message
342             // see https://bugs.eclipse.org/bugs/show_bug.cgi?id=110797
343             containsEmptyTask = true;
344             continue;
345         }
346         // trim the message
347         while (CharOperation.isWhitespace(src[end]) && msgStart <= end)
348             end--;
349         while (CharOperation.isWhitespace(src[msgStart]) && msgStart <= end)
350             msgStart++;
351         // update the end position of the task
352         this.foundTaskPositions[i][1] = end;
353         // get the message source
354         final int messageLength = end - msgStart + 1;
355         char[] message = new char[messageLength];
356         System.arraycopy(src, msgStart, message, 0, messageLength);
357         this.foundTaskMessages[i] = message;
358     }
359     if (containsEmptyTask) {
360         for (int i = foundTaskIndex, max = this.foundTaskCount; i < max; i++) {
361             if (this.foundTaskMessages[i].length == 0) {
362                 loop: for (int j = i + 1; j < max; j++) {
363                     if (this.foundTaskMessages[j].length != 0) {
364                         this.foundTaskMessages[i] = this.foundTaskMessages[j];
365                         this.foundTaskPositions[i][1] = this.foundTaskPositions[j][1];
366                         break loop;
367                     }
368                 }
369             }
370         }
371     }
372 }
373 
374 public char[] getCurrentIdentifierSource() {
375     //return the token REAL source (aka unicodes are precomputed)
376 
377     char[] result;
378     if (this.withoutUnicodePtr != 0) {
379         //0 is used as a fast test flag so the real first char is in position 1
380         System.arraycopy(
381             this.withoutUnicodeBuffer, 
382             1, 
383             result = new char[this.withoutUnicodePtr], 
384             0, 
385             this.withoutUnicodePtr); 
386     } else {
387         int length = this.currentPosition - this.startPosition;
388         if (length == this.eofPosition) return this.source;
389         switch (length) { // see OptimizedLength
390             case 1 :
391                 return optimizedCurrentTokenSource1();
392             case 2 :
393                 return optimizedCurrentTokenSource2();
394             case 3 :
395                 return optimizedCurrentTokenSource3();
396             case 4 :
397                 return optimizedCurrentTokenSource4();
398             case 5 :
399                 return optimizedCurrentTokenSource5();
400             case 6 :
401                 return optimizedCurrentTokenSource6();
402         }
403         //no optimization
404         System.arraycopy(this.source, this.startPosition, result = new char[length], 0, length);
405     }
406     //newIdentCount++;
407     return result;
408 }
409 public int getCurrentTokenEndPosition(){
410     return this.currentPosition - 1;
411 }
412 public char[] getCurrentTokenSource() {
413     // Return the token REAL source (aka unicodes are precomputed)
414 
415     char[] result;
416     if (this.withoutUnicodePtr != 0)
417         // 0 is used as a fast test flag so the real first char is in position 1
418         System.arraycopy(
419             this.withoutUnicodeBuffer, 
420             1, 
421             result = new char[this.withoutUnicodePtr], 
422             0, 
423             this.withoutUnicodePtr); 
424     else {
425         int length;
426         System.arraycopy(
427             this.source, 
428             this.startPosition, 
429             result = new char[length = this.currentPosition - this.startPosition], 
430             0, 
431             length); 
432     }
433     return result;
434 }
435 public final String   getCurrentTokenString() {
436     // Return current token as a string
437 
438     if (this.withoutUnicodePtr != 0) {
439         // 0 is used as a fast test flag so the real first char is in position 1
440         return new String  (
441             this.withoutUnicodeBuffer, 
442             1, 
443             this.withoutUnicodePtr);
444     }
445     return new String  (
446         this.source, 
447         this.startPosition, 
448         this.currentPosition - this.startPosition); 
449 }
450 public char[] getCurrentTokenSourceString() {
451     //return the token REAL source (aka unicodes are precomputed).
452     //REMOVE the two " that are at the beginning and the end.
453 
454     char[] result;
455     if (this.withoutUnicodePtr != 0)
456         //0 is used as a fast test flag so the real first char is in position 1
457         System.arraycopy(this.withoutUnicodeBuffer, 2,
458         //2 is 1 (real start) + 1 (to jump over the ")
459         result = new char[this.withoutUnicodePtr - 2], 0, this.withoutUnicodePtr - 2);
460     else {
461         int length;
462         System.arraycopy(
463             this.source, 
464             this.startPosition + 1, 
465             result = new char[length = this.currentPosition - this.startPosition - 2], 
466             0, 
467             length); 
468     }
469     return result;
470 }
471 public final String   getCurrentStringLiteral() {
472     //return the token REAL source (aka unicodes are precomputed).
473     //REMOVE the two " that are at the beginning and the end.
474 
475     if (this.withoutUnicodePtr != 0)
476         //0 is used as a fast test flag so the real first char is in position 1
477         //2 is 1 (real start) + 1 (to jump over the ")
478         return new String  (this.withoutUnicodeBuffer, 2, this.withoutUnicodePtr - 2);
479     else {
480         return new String  (this.source, this.startPosition + 1, this.currentPosition - this.startPosition - 2);
481     }
482 }
483 public final char[] getRawTokenSource() {
484     int length = this.currentPosition - this.startPosition;
485     char[] tokenSource = new char[length];
486     System.arraycopy(this.source, this.startPosition, tokenSource, 0, length);
487     return tokenSource; 
488 }
489     
490 public final char[] getRawTokenSourceEnd() {
491     int length = this.eofPosition - this.currentPosition - 1;
492     char[] sourceEnd = new char[length];
493     System.arraycopy(this.source, this.currentPosition, sourceEnd, 0, length);
494     return sourceEnd;   
495 }
496     
497 public int getCurrentTokenStartPosition(){
498     return this.startPosition;
499 }
500 /*
501  * Search the source position corresponding to the end of a given line number
502  *
503  * Line numbers are 1-based, and relative to the scanner initialPosition. 
504  * Character positions are 0-based.
505  *
506  * In case the given line number is inconsistent, answers -1.
507  */
508 public final int getLineEnd(int lineNumber) {
509 
510     if (this.lineEnds == null || this.linePtr == -1) 
511         return -1;
512     if (lineNumber > this.lineEnds.length+1) 
513         return -1;
514     if (lineNumber <= 0) 
515         return -1;
516     if (lineNumber == this.lineEnds.length + 1) 
517         return this.eofPosition;
518     return this.lineEnds[lineNumber-1]; // next line start one character behind the lineEnd of the previous line
519 }
520 
521 public final int[] getLineEnds() {
522     //return a bounded copy of this.lineEnds
523     if (this.linePtr == -1) {
524         return EMPTY_LINE_ENDS;
525     }
526     int[] copy;
527     System.arraycopy(this.lineEnds, 0, copy = new int[this.linePtr + 1], 0, this.linePtr + 1);
528     return copy;
529 }
530 
531 /**
532  * Search the source position corresponding to the beginning of a given line number
533  *
534  * Line numbers are 1-based, and relative to the scanner initialPosition. 
535  * Character positions are 0-based.
536  *
537  * e.g. getLineStart(1) --> 0   indicates that the first line starts at character 0.
538  *
539  * In case the given line number is inconsistent, answers -1.
540  * 
541  * @param lineNumber int
542  * @return int
543  */
544 public final int getLineStart(int lineNumber) {
545 
546     if (this.lineEnds == null || this.linePtr == -1) 
547         return -1;
548     if (lineNumber > this.lineEnds.length + 1) 
549         return -1;
550     if (lineNumber <= 0) 
551         return -1;
552     
553     if (lineNumber == 1) 
554         return this.initialPosition;
555     return this.lineEnds[lineNumber-2]+1; // next line start one character behind the lineEnd of the previous line
556 }
557 public final int getNextChar() {
558     try {
559         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
560             && (this.source[this.currentPosition] == 'u')) {
561                 getNextUnicodeChar();
562         } else {
563             this.unicodeAsBackSlash = false;
564             if (this.withoutUnicodePtr != 0) {
565                 unicodeStore();
566             }
567         }
568         return this.currentCharacter;
569     } catch (IndexOutOfBoundsException   e) {
570         return -1;
571     } catch(InvalidInputException e) {
572         return -1;
573     }
574 }
575 public final int getNextCharWithBoundChecks() {
576     if (this.currentPosition >= this.eofPosition) {
577         return -1;
578     }
579     this.currentCharacter = this.source[this.currentPosition++];
580     if (this.currentPosition >= this.eofPosition) {
581         this.unicodeAsBackSlash = false;
582         if (this.withoutUnicodePtr != 0) {
583             unicodeStore();
584         }
585         return this.currentCharacter;
586     }
587     if (this.currentCharacter == '\\' && this.source[this.currentPosition] == 'u') {
588         try {
589             getNextUnicodeChar();
590         } catch (InvalidInputException e) {
591             return -1;
592         }
593     } else {
594         this.unicodeAsBackSlash = false;
595         if (this.withoutUnicodePtr != 0) {
596             unicodeStore();
597         }
598     }
599     return this.currentCharacter;
600 }
601 public final boolean getNextChar(char testedChar) {
602     //BOOLEAN
603     //handle the case of unicode.
604     //when a unicode appears then we must use a buffer that holds char internal values
605     //At the end of this method currentCharacter holds the new visited char
606     //and currentPosition points right next after it
607     //Both previous lines are true if the currentCharacter is == to the testedChar
608     //On false, no side effect has occured.
609 
610     //ALL getNextChar.... ARE OPTIMIZED COPIES 
611 
612     if (this.currentPosition >= this.eofPosition) { // handle the obvious case upfront
613         this.unicodeAsBackSlash = false;
614         return false;
615     }
616 
617     int temp = this.currentPosition;
618     try {
619         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
620             && (this.source[this.currentPosition] == 'u')) {
621             getNextUnicodeChar();
622             if (this.currentCharacter != testedChar) {
623                 this.currentPosition = temp;
624                 this.withoutUnicodePtr--;
625                 return false;
626             }
627             return true;
628         } //-------------end unicode traitement--------------
629         else {
630             if (this.currentCharacter != testedChar) {
631                 this.currentPosition = temp;
632                 return false;
633             }
634             this.unicodeAsBackSlash = false;
635             if (this.withoutUnicodePtr != 0)
636                 unicodeStore();
637             return true;
638         }
639     } catch (IndexOutOfBoundsException   e) {
640         this.unicodeAsBackSlash = false;
641         this.currentPosition = temp;
642         return false;
643     } catch(InvalidInputException e) {
644         this.unicodeAsBackSlash = false;
645         this.currentPosition = temp;
646         return false;
647     }
648 }
649 public final int getNextChar(char testedChar1, char testedChar2) {
650     //INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others
651     //test can be done with (x==0) for the first and (x>0) for the second
652     //handle the case of unicode.
653     //when a unicode appears then we must use a buffer that holds char internal values
654     //At the end of this method currentCharacter holds the new visited char
655     //and currentPosition points right next after it
656     //Both previous lines are true if the currentCharacter is == to the testedChar1/2
657     //On false, no side effect has occured.
658 
659     //ALL getNextChar.... ARE OPTIMIZED COPIES 
660     if (this.currentPosition >= this.eofPosition) // handle the obvious case upfront
661         return -1;
662 
663     int temp = this.currentPosition;
664     try {
665         int result;
666         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
667             && (this.source[this.currentPosition] == 'u')) {
668             getNextUnicodeChar();
669             if (this.currentCharacter == testedChar1) {
670                 result = 0;
671             } else if (this.currentCharacter == testedChar2) {
672                 result = 1;
673             } else {
674                 this.currentPosition = temp;
675                 this.withoutUnicodePtr--;
676                 result = -1;
677             }
678             return result;
679         } else {
680             if (this.currentCharacter == testedChar1) {
681                 result = 0;
682             } else if (this.currentCharacter == testedChar2) {
683                 result = 1;
684             } else {
685                 this.currentPosition = temp;
686                 return -1;
687             }
688 
689             if (this.withoutUnicodePtr != 0)
690                 unicodeStore();
691             return result;
692         }
693     } catch (IndexOutOfBoundsException   e) {
694         this.currentPosition = temp;
695         return -1;
696     } catch(InvalidInputException e) {
697         this.currentPosition = temp;
698         return -1;
699     }
700 }
701 public final boolean getNextCharAsDigit() throws InvalidInputException {
702     //BOOLEAN
703     //handle the case of unicode.
704     //when a unicode appears then we must use a buffer that holds char internal values
705     //At the end of this method currentCharacter holds the new visited char
706     //and currentPosition points right next after it
707     //Both previous lines are true if the currentCharacter is a digit
708     //On false, no side effect has occured.
709 
710     //ALL getNextChar.... ARE OPTIMIZED COPIES 
711     if (this.currentPosition >= this.eofPosition) // handle the obvious case upfront
712         return false;
713 
714     int temp = this.currentPosition;
715     try {
716         if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
717             && (this.source[this.currentPosition] == 'u')) {
718             getNextUnicodeChar();
719             if (!ScannerHelper.isDigit(this.currentCharacter)) {
720                 this.currentPosition = temp;
721                 this.withoutUnicodePtr--;
722                 return false;
723             }
724             return true;
725         } else {
726             if (!ScannerHelper.isDigit(this.currentCharacter)) {
727                 this.currentPosition = temp;
728                 return false;
729             }
730             if (this.withoutUnicodePtr != 0)
731                 unicodeStore();
732             return true;
733         }
734     } catch (IndexOutOfBoundsException   e) {
735         this.currentPosition = temp;
736         return false;
737     } catch(InvalidInputException e) {
738         this.currentPosition = temp;
739         return false;
740     }
741 }
742 public final boolean getNextCharAsDigit(int radix) {
743     //BOOLEAN
744     //handle the case of unicode.
745     //when a unicode appears then we must use a buffer that holds char internal values
746     //At the end of this method currentCharacter holds the new visited char
747     //and currentPosition points right next after it
748     //Both previous lines are true if the currentCharacter is a digit base on radix
749     //On false, no side effect has occured.
750 
751     //ALL getNextChar.... ARE OPTIMIZED COPIES 
752     if (this.currentPosition >= this.eofPosition) // handle the obvious case upfront
753         return false;
754 
755     int temp = this.currentPosition;
756     try