KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > netbeans > api > lexer > TokenSequence


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */
19
20 package org.netbeans.api.lexer;
21
22 import java.util.ConcurrentModificationException JavaDoc;
23 import org.netbeans.lib.lexer.EmbeddingContainer;
24 import org.netbeans.lib.lexer.SubSequenceTokenList;
25 import org.netbeans.lib.lexer.LexerUtilsConstants;
26 import org.netbeans.lib.lexer.TokenList;
27 import org.netbeans.lib.lexer.inc.FilterSnapshotTokenList;
28 import org.netbeans.lib.lexer.inc.SnapshotTokenList;
29 import org.netbeans.lib.lexer.token.AbstractToken;
30
31 /**
32  * Token sequence allows to iterate between tokens
33  * of a token hierarchy.
34  * <br/>
35  * Token sequence for top-level language of a token hierarchy
36  * may be obtained by {@link TokenHierarchy#tokenSequence()}.
37  *
38  * <p>
39  * Use of token sequence is a two-step operation:
40  * <ol>
41  * <li>
42  * Position token sequence before token that should first be retrieved
43  * (or behind desired token when iterating backwards).
44  * <br/>
45  * One of the following ways may be used:
46  * <ul>
47  * <li> {@link #move(int)} positions TS before token that either starts
48  * at the given offset or "contains" it.
49  * </li>
50  * <li> {@link #moveIndex(int)} positions TS before n-th token in the underlying
51  * token list.
52  * </li>
53  * <li> {@link #moveStart()} positions TS before the first token. </li>
54  * <li> {@link #moveEnd()} positions TS behind the last token. </li>
55  * <li> Do nothing - TS is positioned before the first token automatically by default. </li>
56  * </ul>
57  * Token sequence will always be positioned between tokens
58  * when using one of the operations above
59  * ({@link #token()} will return <code>null</code> to signal between-tokens location).
60  * <br/>
61  * </li>
62  *
63  * <li>
64  * Start iterating through the tokens in forward/backward direction
65  * by using {@link #moveNext()} or {@link #movePrevious()}.
66  * <br/>
67  * If <code>moveNext()</code> or <code>movePrevious()</code> returned
68  * <code>true</code> then TS is positioned
69  * over a concrete token retrievable by {@link #token()}.
70  * <br/>
71  * Its offset can be retrieved by {@link #offset()}.
72  * </li>
73  * </ol>
74  * </p>
75  *
76  * <p>
77  * An example of forward iteration through the tokens:
78  * <pre>
79  * TokenSequence ts = tokenHierarchy.tokenSequence();
80  * // Possible positioning by ts.move(offset) or ts.moveIndex(index)
81  * while (ts.moveNext()) {
82  * Token t = ts.token();
83  * if (t.id() == ...) { ... }
84  * if (TokenUtilities.equals(t.text(), "mytext")) { ... }
85  * if (ts.offset() == ...) { ... }
86  * }
87  * </pre>
88  * </p>
89  *
90  * <p>
91  * This class should be used by a single thread only.
92  * </p>
93  *
94  * @author Miloslav Metelka
95  * @version 1.00
96  */
97
98 public final class TokenSequence<T extends TokenId> {
99     
100     private TokenList<T> tokenList; // 8 + 4 = 12 bytes
101
102     private AbstractToken<T> token; // 16 bytes
103
104     private int tokenIndex; // 20 bytes
105
106     /**
107      * Offset in the input at which the current token is located
108      * or <code>-1</code> if the offset needs to be computed.
109      */
110     private int tokenOffset = -1; // 24 bytes
111
112     /**
113      * Copy of the modCount of the token list. If the token list's modCount
114      * changes (by modification) this token sequence will become invalid.
115      */
116     private final int modCount; // 28 bytes
117
118     /**
119      * Parent token indexes allow to effectively determine parent tokens
120      * in the tree token hierarchy.
121      * <br/>
122      * The first index corresponds to the top language in the hierarchy
123      * and the ones that follow point to subsequent embedded levels.
124      */
125     private int[] parentTokenIndexes; // 32 bytes
126
127     /**
128      * Package-private constructor used by API accessor.
129      */
130     TokenSequence(TokenList<T> tokenList) {
131         this.tokenList = tokenList;
132         this.modCount = tokenList.modCount();
133     }
134
135     /**
136      * Get the language describing token ids
137      * used by tokens in this token sequence.
138      */
139     public Language<T> language() {
140         return LexerUtilsConstants.mostEmbeddedLanguage(languagePath());
141     }
142
143     /**
144      * Get the complete language path of the tokens contained
145      * in this token sequence.
146      */
147     public LanguagePath languagePath() {
148         return tokenList.languagePath();
149     }
150
151     /**
152      * Get token to which this token sequence points to or null
153      * if TS is positioned between tokens
154      * ({@link #moveNext()} or {@link #movePrevious()} were not called yet).
155      * <br/>
156      * A typical iteration usage:
157      * <pre>
158      * TokenSequence ts = tokenHierarchy.tokenSequence();
159      * // Possible positioning by ts.move(offset) or ts.moveIndex(index)
160      * while (ts.moveNext()) {
161      * Token t = ts.token();
162      * if (t.id() == ...) { ... }
163      * if (TokenUtilities.equals(t.text(), "mytext")) { ... }
164      * if (ts.offset() == ...) { ... }
165      * }
166      * </pre>
167      *
168      * The returned token instance may be flyweight
169      * ({@link Token#isFlyweight()} returns true)
170      * which means that its {@link Token#offset(TokenHierarchy)} will return -1.
171      * <br/>
172      * To find a correct offset use {@link #offset()}.
173      * <br/>
174      * Or if its necessary to revert to a regular non-flyweigt token
175      * the {@link #offsetToken()} may be used.
176      * </p>
177      *
178      * <p>
179      * The lifetime of the returned token instance may be limited for mutable inputs.
180      * The token instance should not be held across the input source modifications.
181      * </p>
182      *
183      * @return token instance to which this token sequence is currently positioned
184      * or null if this token sequence is not positioned to any token which may
185      * happen after TS creation or after use of {@link #move(int)} or {@link #moveIndex(int)}.
186      *
187      * @see #offsetToken()
188      */
189     public Token<T> token() {
190         return token;
191     }
192     
193     /**
194      * Similar to {@link #token()} but always returns a non-flyweight token
195      * with the appropriate offset.
196      * <br/>
197      * If the current token is flyweight then this method replaces it
198      * with the corresponding non-flyweight token which it then returns.
199      * <br/>
200      * Subsequent calls to {@link #token()} will also return this non-flyweight token.
201      *
202      * <p>
203      * This method may be handy if the token instance is referenced in a standalone way
204      * (e.g. in an expression node of a parse tree) and it's necessary
205      * to get the appropriate offset from the token itself
206      * later when a token sequence will not be available.
207      * </p>
208      * @throws IllegalStateException if {@link #token()} returns null.
209      */
210     public Token<T> offsetToken() {
211         checkTokenNotNull();
212         if (token.isFlyweight()) {
213             token = tokenList.replaceFlyToken(tokenIndex, token, offset());
214         }
215         return token;
216     }
217     
218     /**
219      * Get the offset of the current token in the underlying input.
220      * <br>
221      * The token's offset should never be computed by a client of the token sequence
222      * by adding/subtracting tokens' length to a client's variable because
223      * in case of the immutable token sequences there can be gaps
224      * between tokens if some tokens get filtered out.
225      * <br>
226      * Instead this method should always be used because it offers
227      * best performance with a constant time complexity.
228      *
229      * @return &gt;=0 absolute offset of the current token in the underlying input.
230      * @throws IllegalStateException if {@link #token()} returns null.
231      */
232     public int offset() {
233         checkTokenNotNull();
234         if (tokenOffset == -1) {
235             tokenOffset = tokenList.tokenOffset(tokenIndex);
236         }
237         return tokenOffset;
238     }
239     
240     /**
241      * Get an index of token to which (or before which) this TS is currently positioned.
242      * <br/>
243      * <p>
244      * Initially or after {@link #move(int)} or {@link #moveIndex(int)}
245      * token sequence is positioned between tokens:
246      * <pre>
247      * Token[0] Token[1] ... Token[n]
248      * ^ ^ ^
249      * Index: 0 1 n
250      * </pre>
251      * </p>
252      *
253      * <p>
254      * After use of {@link #moveNext()} or {@link #movePrevious()}
255      * the token sequence is positioned over one of the actual tokens:
256      * <pre>
257      * Token[0] Token[1] ... Token[n]
258      * ^ ^ ^
259      * Index: 0 1 n
260      * </pre>
261      * </p>
262      *
263      * @return &gt;=0 index of token to which (or before which) this TS is currently positioned.
264      */
265     public int index() {
266         return tokenIndex;
267     }
268
269     /**
270      * Get embedded token sequence if the token
271      * to which this token sequence is currently positioned
272      * has a language embedding.
273      * <br/>
274      * If there is a custom embedding created by
275      * {@link #createEmbedding(Language,int,int)} it will be returned
276      * instead of the default embedding
277      * (the one created by <code>LanguageHierarchy.embedding()</code>
278      * or <code>LanguageProvider</code>).
279      *
280      * @return embedded sequence or null if no embedding exists for this token.
281      * @throws IllegalStateException if {@link #token()} returns null.
282      */
283     public TokenSequence<? extends TokenId> embedded() {
284         checkTokenNotNull();
285         return embeddedImpl(null);
286     }
287     
288     private <ET extends TokenId> TokenSequence<ET> embeddedImpl(Language<ET> embeddedLanguage) {
289         TokenList<ET> embeddedTokenList
290                 = EmbeddingContainer.getEmbedding(tokenList, tokenIndex, embeddedLanguage);
291         if (embeddedTokenList != null) {
292             TokenList<T> tl = tokenList;
293             if (tokenList.getClass() == SubSequenceTokenList.class) {
294                 tl = ((SubSequenceTokenList<T>)tokenList).delegate();
295             }
296
297             if (tl.getClass() == FilterSnapshotTokenList.class) {
298                 embeddedTokenList = new FilterSnapshotTokenList<ET>(embeddedTokenList,
299                         ((FilterSnapshotTokenList<T>)tl).tokenOffsetDiff());
300
301             } else if (tl.getClass() == SnapshotTokenList.class) {
302                 embeddedTokenList = new FilterSnapshotTokenList<ET>(embeddedTokenList,
303                         offset() - token().offset(null));
304             }
305             return new TokenSequence<ET>(embeddedTokenList);
306
307         } else // Embedded token list does not exist
308 return null;
309     }
310
311     /**
312      * Get embedded token sequence if the token
313      * to which this token sequence is currently positioned
314      * has a language embedding.
315      *
316      * @throws IllegalStateException if {@link #token()} returns null.
317      */
318     public <ET extends TokenId> TokenSequence<ET> embedded(Language<ET> embeddedLanguage) {
319         checkTokenNotNull();
320         return embeddedImpl(embeddedLanguage);
321     }
322
323     /**
324      * Create language embedding without joining of the embedded sections.
325      *
326      * @throws IllegalStateException if {@link #token()} returns null.
327      * @see #createEmbedding(Language, int, int, boolean)
328      */
329     public boolean createEmbedding(Language<? extends TokenId> embeddedLanguage,
330     int startSkipLength, int endSkipLength) {
331         return createEmbedding(embeddedLanguage, startSkipLength, endSkipLength, false);
332     }
333
334     /**
335      * Create language embedding described by the given parameters.
336      * <br/>
337      * If the underying text input is mutable then this method should only be called
338      * within a read lock over the text input.
339      *
340      * @param embeddedLanguage non-null embedded language
341      * @param startSkipLength &gt;=0 number of characters in an initial part of the token
342      * for which the language embedding is defined that should be excluded
343      * from the embedded section. The excluded characters will not be lexed
344      * and there will be no tokens created for them.
345      * @param endSkipLength &gt;=0 number of characters at the end of the token
346      * for which the language embedding is defined that should be excluded
347      * from the embedded section. The excluded characters will not be lexed
348      * and there will be no tokens created for them.
349      * @param joinSections whether sections with this embedding should be joined
350      * across the input source or whether they should stay separate.
351      * <br/>
352      * For example for HTML sections embedded in JSP this flag should be true:
353      * <pre>
354      * &lt;!-- HTML comment start
355      * &lt;% System.out.println("Hello"); %&gt;
356             still in HTML comment --&lt;
357      * </pre>
358      * <br/>
359      * Only the embedded sections with the same language path can be joined.
360      * @return true if the embedding was created successfully or false if an embedding
361      * with the given language already exists for this token.
362      * @throws IllegalStateException if {@link #token()} returns null.
363      */
364     public boolean createEmbedding(Language<? extends TokenId> embeddedLanguage,
365     int startSkipLength, int endSkipLength, boolean joinSections) {
366         checkTokenNotNull();
367         return EmbeddingContainer.createEmbedding(tokenList, tokenIndex,
368                 embeddedLanguage, startSkipLength, endSkipLength, joinSections);
369     }
370
371     /**
372      * Move to the next token in this token sequence.
373      *
374      * <p>
375      * The next token may not necessarily start at the offset where
376      * the previous token ends (there may be gaps between tokens
377      * caused by token filtering). {@link #offset()} should be used
378      * for offset retrieval.
379      * </p>
380      *
381      * @return true if the sequence was successfully moved to the next token
382      * or false if it was not moved before there are no more tokens
383      * in the forward direction.
384      * @throws ConcurrentModificationException if this token sequence
385      * is no longer valid because of an underlying mutable input source modification.
386      */
387     public boolean moveNext() {
388         checkModCount();
389         if (token != null) // Token already fetched
390 tokenIndex++;
391         Object JavaDoc tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenIndex);
392         if (tokenOrEmbeddingContainer != null) {
393             AbstractToken origToken = token;
394             token = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
395             // If origToken == null then the right offset might already be pre-computed from move()
396 if (tokenOffset != -1) {
397                 if (origToken != null) {
398                     // If the token list is continuous or the fetched token
399 // is flyweight (there cannot be a gap before flyweight token)
400 // the original offset can be just increased
401 // by the original token's length.
402 if (tokenList.isContinuous() || token.isFlyweight()) {
403                         tokenOffset += origToken.length(); // advance by previous token's length
404 } else // Offset must be recomputed
405 tokenOffset = -1; // mark the offset to be recomputed
406 } else // Not valid token previously
407 tokenOffset = -1;
408             }
409             return true;
410         }
411         if (token != null) // Unsuccessful move from existing token
412 tokenIndex--;
413         return false;
414     }
415
416     /**
417      * Move to a previous token in this token sequence.
418      *
419      * <p>
420      * The previous token may not necessarily end at the offset where
421      * the previous token started (there may be gaps between tokens
422      * caused by token filtering). {@link #offset()} should be used
423      * for offset retrieval.
424      * </p>
425      *
426      * @return true if the sequence was successfully moved to the previous token
427      * or false if it was not moved because there are no more tokens
428      * in the backward direction.
429      * @throws ConcurrentModificationException if this token sequence
430      * is no longer valid because of an underlying mutable input source modification.
431      */
432     public boolean movePrevious() {
433         checkModCount();
434         if (tokenIndex > 0) {
435             AbstractToken origToken = token;
436             tokenIndex--;
437             token = LexerUtilsConstants.token(tokenList.tokenOrEmbeddingContainer(tokenIndex));
438             if (tokenOffset != -1) {
439                 // If the token list is continuous or the original token
440 // is flyweight (there cannot be a gap before flyweight token)
441 // the original offset can be just decreased
442 // by the fetched token's length.
443 if (tokenList.isContinuous() || origToken.isFlyweight()) {
444                     tokenOffset -= token.length(); // decrease by the fetched's token length
445 } else { // mark the offset to be computed upon call to offset()
446 tokenOffset = -1;
447                 }
448             }
449             return true;
450
451         } // no tokens below index zero
452 return false;
453     }
454
455     /**
456      * Position token sequence between <code>index-1</code>
457      * and <code>index</code> tokens.
458      * <br/>
459      * TS will be positioned in the following way:
460      * <pre>
461      * Token[0] ... Token[index-1] Token[index] ...
462      * ^ ^ ^
463      * Index: 0 index-1 index
464      * </pre>
465      *
466      * <p>
467      * Subsequent {@link #moveNext()} or {@link #movePrevious()} is needed to fetch
468      * a concrete token in the desired direction.
469      * <br/>
470      * Subsequent {@link #moveNext()} will position TS over <code>Token[index]</code>
471      * (or {@link #movePrevious()} will position TS over <code>Token[index-1]</code>)
472      * so that <code>{@link #token()} != null</code>.
473      *
474      * @param index index of the token to which this sequence
475      * should be positioned.
476      * <br/>
477      * If <code>index >= {@link #tokenCount()}</code>
478      * then the TS will be positioned to {@link #tokenCount()}.
479      * <br/>
480      * If <code>index < 0</code> then the TS will be positioned to index 0.
481      *
482      * @return difference between requested index and the index to which TS
483      * is really set.
484      * @throws ConcurrentModificationException if this token sequence
485      * is no longer valid because of an underlying mutable input source modification.
486      */
487     public int moveIndex(int index) {
488         checkModCount();
489         if (index >= 0) {
490             Object JavaDoc tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index);
491             if (tokenOrEmbeddingContainer != null) { // enough tokens
492 resetTokenIndex(index);
493             } else // Token at the requested index does not exist - leave orig. index
494 resetTokenIndex(tokenCount());
495         } else // index < 0
496 resetTokenIndex(0);
497         return index - tokenIndex;
498     }
499     
500     /**
501      * Move the token sequence to be positioned before the first token.
502      * <br/>
503      * This is equivalent to <code>moveIndex(0)</code>.
504      */
505     public void moveStart() {
506         moveIndex(0);
507     }
508     
509     /**
510      * Move the token sequence to be positioned behind the last token.
511      * <br/>
512      * This is equivalent to <code>moveIndex(tokenCount())</code>.
513      */
514     public void moveEnd() {
515         moveIndex(tokenCount());
516     }
517     
518     /**
519      * Move token sequence to be positioned between <code>index-1</code>
520      * and <code>index</code> tokens where Token[index] either starts at offset
521      * or "contains" the offset.
522      * <br/>
523      * <pre>
524      * +----------+-----+----------------+--------------+------
525      * | Token[0] | ... | Token[index-1] | Token[index] | ...
526      * | "public" | ... | "static" | "int" | ...
527      * +----------+-----+----------------+--------------+------
528      * ^ ^ ^
529      * Index: 0 index-1 index
530      * Offset: ---^ (if offset points to 'i','n' or 't')
531      * </pre>
532      *
533      * <p>
534      * Subsequent {@link #moveNext()} or {@link #movePrevious()} is needed to fetch
535      * a concrete token.
536      * <br/>
537      * If the offset is too big then the token sequence will be positioned
538      * behind the last token.
539      * </p>
540      *
541      * <p>
542      * If token filtering is used there may be gaps that are not covered
543      * by any tokens and if the offset is contained in such gap then
544      * the token sequence will be positioned before the token that follows the gap.
545      * </p>
546      *
547      *
548      * @param offset absolute offset to which the token sequence should be moved.
549      * @return difference between the reqeuested offset
550      * and the start offset of the token
551      * before which the the token sequence gets positioned.
552      *
553      * @throws ConcurrentModificationException if this token sequence
554      * is no longer valid because of an underlying mutable input source modification.
555      */
556     public int move(int offset) {
557         checkModCount();
558         // Token count in the list may change as possibly other threads
559 // keep asking for tokens. Root token list impls create tokens lazily
560 // when asked by clients.
561 int tokenCount = tokenList.tokenCountCurrent(); // presently created token count
562 if (tokenCount == 0) { // no tokens yet -> attempt to create at least one
563 if (tokenList.tokenOrEmbeddingContainer(0) == null) { // really no tokens at all
564 // In this case the token sequence could not be positioned yet
565 // so no need to reset "index" or other vars
566 resetTokenIndex(0);
567                 return offset;
568             }
569             // Re-get the present token count (could be created a chunk of tokens at once)
570 tokenCount = tokenList.tokenCountCurrent();
571         }
572
573         // tokenCount surely >0
574 int prevTokenOffset = tokenList.tokenOffset(tokenCount - 1);
575         if (offset > prevTokenOffset) { // may need to create further tokens if they do not exist
576 // Force token list to create subsequent tokens
577 // Cannot subtract offset by each token's length because
578 // there may be gaps between tokens due to token id filter use.
579 int tokenLength = LexerUtilsConstants.token(tokenList, tokenCount - 1).length();
580             while (offset >= prevTokenOffset + tokenLength) { // above present token
581 Object JavaDoc tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenCount);
582                 if (tokenOrEmbeddingContainer != null) {
583                     AbstractToken t = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
584                     if (t.isFlyweight()) { // need to use previous tokenLength
585 prevTokenOffset += tokenLength;
586                     } else { // non-flyweight token - retrieve offset
587 prevTokenOffset = tokenList.tokenOffset(tokenCount);
588                     }
589                     tokenLength = t.length();
590                     tokenCount++;
591
592                 } else { // no more tokens => position behind last token
593 resetTokenIndex(tokenCount);
594                     tokenOffset = prevTokenOffset + tokenLength; // May assign the token's offset in advance
595 return offset - tokenOffset;
596                 }
597             }
598             resetTokenIndex(tokenCount - 1);
599             tokenOffset = prevTokenOffset; // May assign the token's offset in advance
600 return offset - prevTokenOffset;
601         }
602         
603         // The offset is within the currently recognized tokens
604 // Use binary search
605 int low = 0;
606         int high = tokenCount - 1;
607         
608         while (low <= high) {
609             int mid = (low + high) / 2;
610             int midStartOffset = tokenList.tokenOffset(mid);
611             
612             if (midStartOffset < offset) {
613                 low = mid + 1;
614             } else if (midStartOffset > offset) {
615                 high = mid - 1;
616             } else {
617                 // Token starting exactly at offset found
618 resetTokenIndex(mid);
619                 tokenOffset = midStartOffset;
620                 return 0; // right at the token begining
621 }
622         }
623         
624         // Not found exactly and high + 1 == low => high < low
625 // BTW there may be gaps between tokens; if offset is in gap then position to higher token
626 if (high >= 0) { // could be -1
627 AbstractToken t = LexerUtilsConstants.token(tokenList, high);
628             prevTokenOffset = tokenList.tokenOffset(high);
629             // If gaps allowed check whether the token at "high" contains the offset
630 if (!tokenList.isContinuous() && offset > prevTokenOffset + t.length()) {
631                 // Offset in the gap above the "high" token
632 high++;
633                 prevTokenOffset += t.length();
634             }
635         } else { // at least one token exists => use token at index 0
636 high = 0;
637             prevTokenOffset = tokenList.tokenOffset(0); // result may differ from 0
638 }
639         resetTokenIndex(high);
640         tokenOffset = prevTokenOffset;
641         return offset - prevTokenOffset;
642     }
643     
644     /**
645      * Check whether this TS contains zero tokens.
646      * <br/>
647      * This check is strongly preferred over <code>tokenCount() == 0</code>.
648      *
649      * @see #tokenCount()
650      */
651     public boolean isEmpty() {
652         return (tokenIndex == 0 && tokenList.tokenOrEmbeddingContainer(0) == null);
653     }
654
655     /**
656      * Return total count of tokens in this sequence.
657      * <br>
658      * <b>Note:</b> Calling this method will lead
659      * to creation of all the remaining tokens in the sequence
660      * if they were not yet created.
661      *
662      * @return total number of tokens in this token sequence.
663      */
664     public int tokenCount() {
665         checkModCount();
666         return tokenList.tokenCount();
667     }
668     
669     /**
670      * Create sub sequence of this token sequence that only returns
671      * tokens above the given offset.
672      *
673      * @param startOffset only tokens satisfying
674      * <code>tokenStartOffset + tokenLength > startOffset</code>
675      * will be present in the returned sequence.
676      * @return non-null sub sequence of this token sequence.
677      */
678     public TokenSequence<T> subSequence(int startOffset) {
679         return subSequence(startOffset, Integer.MAX_VALUE);
680     }
681     
682     /**
683      * Create sub sequence of this token sequence that only returns
684      * tokens between the given offsets.
685      *
686      * @param startOffset only tokens satisfying
687      * <code>tokenStartOffset + tokenLength > startOffset</code>
688      * will be present in the returned sequence.
689      * @param endOffset >=startOffset only tokens satisfying
690      * <code>tokenStartOffset < endOffset</code>
691      * will be present in the returned sequence.
692      * @return non-null sub sequence of this token sequence.
693      */
694     public TokenSequence<T> subSequence(int startOffset, int endOffset) {
695         checkModCount(); // Ensure subsequences on valid token sequences only
696 TokenList<T> tl;
697         if (tokenList.getClass() == SubSequenceTokenList.class) {
698             SubSequenceTokenList<T> stl = (SubSequenceTokenList<T>)tokenList;
699             tl = stl.delegate();
700             startOffset = Math.max(startOffset, stl.limitStartOffset());
701             endOffset = Math.min(endOffset, stl.limitEndOffset());
702         } else // Regular token list
703 tl = tokenList;
704         return new TokenSequence<T>(new SubSequenceTokenList<T>(tl, startOffset, endOffset));
705     }
706     
707     public String JavaDoc toString() {
708         return LexerUtilsConstants.appendTokenList(null, tokenList, tokenIndex).toString();
709     }
710     
711     int[] parentTokenIndexes() {
712         return parentTokenIndexes;
713     }
714
715     private void resetTokenIndex(int index) {
716         // Position to the given index e.g. by move() and moveIndex()
717 tokenIndex = index;
718         token = null;
719         tokenOffset = -1;
720     }
721
722     private void checkTokenNotNull() {
723         if (token == null) {
724             throw new IllegalStateException JavaDoc(
725                 "No token fetched by moveNext() from token sequence yet: index=" + tokenIndex
726             ); // NOI18N
727 }
728     }
729     
730     private void checkModCount() {
731         if (tokenList.modCount() != this.modCount) {
732             throw new ConcurrentModificationException JavaDoc(
733                 "This token sequence is no longer valid. Underlying token hierarchy" // NOI18N
734 + " has been modified: " + this.modCount + " != " + tokenList.modCount() // NOI18N
735 );
736         }
737     }
738
739 }
Popular Tags