KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > netbeans > lib > lexer > TokenList


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */

19
20 package org.netbeans.lib.lexer;
21
22 import java.util.Set JavaDoc;
23 import org.netbeans.api.lexer.LanguagePath;
24 import org.netbeans.api.lexer.InputAttributes;
25 import org.netbeans.api.lexer.TokenId;
26 import org.netbeans.lib.lexer.token.AbstractToken;
27
28 /**
29  * Browsable list of tokens.
30  * <br>
31  * {@link org.netbeans.api.lexer.TokenSequence} delegates
32  * all its operation to this class so any service provider
33  * delivering this class will be able to produce token sequences.
34  *
35  * There are various implementations of the token list:
36  * <ul>
37  * <li>BatchTokenList</li> - predecessor of batch token lists
38  * <li>TextTokenList</li> - token list over immutable char sequence
39  * <li>CopyTextTokenList</li> - token list over text input
40  * that needs to be copied. Characters that belong to tokens
41  * skipped due to skipTokenIds do not need to be copied.
42  * <li>SkimTokenList</li> - filter over CopyTextTokenList
43  * to store the token characters in multiple arrays
44  * and to correctly compute the tokens' starting offsets.
45  * <li>IncTokenList</li> - token list for mutable-input environment.
46  * <li>EmbeddedTokenList</li> - token list for a single language embedding
47  * suitable for both batch and incremental environments.
48  * </ul>
49  *
50  * @author Miloslav Metelka
51  * @version 1.00
52  */

53
54 public interface TokenList<T extends TokenId> {
55     
56     /**
57      * Language path of this token list.
58      */

59     LanguagePath languagePath();
60     
61     /**
62      * Get token or {@link EmbeddingContainer} at given index in this list.
63      * <br/>
64      * The method's implementation may need to be synchronized as multiple
65      * threads can access it at the same time.
66      * <br/>
67      * The requested index value may be arbitrarily high
68      * (e.g. when TokenSequence.move(index) is used for too high value).
69      *
70      * @param &gt;=0 index of the token in this list.
71      * @return valid token or null if the index is too high.
72      */

73     Object JavaDoc tokenOrEmbeddingContainer(int index);
74
75     /**
76      * Replace flyweight token at the given index with its non-flyweight copy.
77      * <br/>
78      * This may be requested by <code>TokenSequence.offsetToken()</code>.
79      *
80      * @param index &gt;=0 index of the flyweight token in this list.
81      * @param flyToken non-null flyweight token.
82      * @param offset >=0 absolute offset where the flyweight token resides.
83      * @return non-flyweight token instance.
84      */

85     AbstractToken<T> replaceFlyToken(int index, AbstractToken<T> flyToken, int offset);
86     
87     /**
88      * Wrap the token by a branch token list due to language embedding
89      * that exists for the token.
90      *
91      * @param index existing index in this token list at which the token
92      * should be wrapped with the embedding info.
93      * @param embeddingContainer embedding info that should wrap the token.
94      */

95     void wrapToken(int index, EmbeddingContainer<T> embeddingContainer);
96     
97     /**
98      * Get absolute offset of the token at the given index in the token list.
99      * <br>
100      * This method can only be called if the token at the given index
101      * was already fetched by {@link tokenOrEmbeddingContainer(int)}.
102      * <br/>
103      * For branch token lists this method is only expected to be called
104      * after {@link #updateStartOffsetShift()} was called so it does not perform
105      * any checking whether the start offset of the token list is up-to-date.
106      */

107     int tokenOffset(int index);
108     
109     /**
110      * Get total count of tokens in the list.
111      * <br/>
112      * For token lists that create the tokens lazily
113      * this will lead to lexing till the end of the input.
114      */

115     int tokenCount();
116
117     /**
118      * Return present number of tokens in the token list but do not create
119      * any new tokens (because of possible lazy token creation).
120      * <br/>
121      * This is necessary e.g. for <code>TokenSequence.move()</code>
122      * that needs a binary search for fast positioning
123      * but using {@link #tokenCount()} would lead to unnecessary creation
124      * of all tokens.
125      */

126     int tokenCountCurrent();
127
128     /**
129      * Get number of modifications which mutated this token list.
130      * <br>
131      * Token sequence remembers this number when it gets constructed
132      * and checks this number when it moves between tokens
133      * and if there is an extra modification performed it throws
134      * <code>IllegalStateException</code>.
135      *
136      * <p>
137      * This is also used to check whether this token list corresponds to mutable input
138      * or not because unmodifiable lists return -1 from this method.
139      *
140      * <p>
141      * For branch token lists the {@link #updateStartOffsetShift()} ensures
142      * that the value returned by this method is most up-to-date
143      * (equals to the root list's one).
144      *
145      * @return number of modifications performed to the list.
146      * <br/>
147      * Returns -1 if this list is constructed for immutable input and cannot be mutated.
148      */

149     int modCount();
150     
151     /**
152      * Get absolute offset of the child token with the given raw offset
153      * in the underlying input.
154      *
155      * @param rawOffset raw offset of the child token.
156      * @return absolute offset in the input.
157      */

158     int childTokenOffset(int rawOffset);
159     
160     /**
161      * Get character of a token from the character sequence represented
162      * by this support.
163      *
164      * @param rawOffset raw offset of the child token.
165      * The given offset value may need to be preprocessed before using (it depends
166      * on a nature of the token list).
167      * @param index index inside the token's text that should be returned.
168      * This value cannot be simply added to the previous parameter
169      * for mutable token lists as the value could errorneously point
170      * into a middle of the offset gap then.
171      * @return appropriate character that the token has requested.
172      */

173     char childTokenCharAt(int rawOffset, int index);
174
175     /**
176      * Get the root token list of the token list hierarchy.
177      */

178     TokenList<? extends TokenId> root();
179     
180     /**
181      * Get token hierarchy operation for this token list or null
182      * if this token list does not have any token hierarchy.
183      */

184     TokenHierarchyOperation<?,? extends TokenId> tokenHierarchyOperation();
185     
186     /**
187      * Extra attributes related to the input being lexed.
188      */

189     InputAttributes inputAttributes();
190
191     /**
192      * Get lookahead information for the token at the existing token index.
193      * <br/>
194      * Lookahead is number of characters that the lexer has read
195      * past the end of the given token in order to recognize it in the text.
196      * <br>
197      * This information allows the lexer to know whether modifications
198      * past the end of the token can affect its validity.
199      *
200      * <p>
201      * In general only mutable token lists benefit from this information
202      * but non-mutable token lists may store the information as well for testing
203      * purposes.
204      * </p>
205      *
206      * @param index index of the existing token.
207      * @return &gt;=0 number of characters that the lexer has read
208      * in order to recognize this token. Return zero if this token list
209      * does not maintain lookaheads.
210      */

211     int lookahead(int index);
212     
213     /**
214      * Get state information for the token at the existing token index.
215      * <br/>
216      * It is an object defining lexer's state after recognition
217      * of the given token.
218      * <br/>
219      * This information allows to restart the lexer at the end of the given token.
220      *
221      * <p>
222      * In general only mutable token lists benefit from this information
223      * but non-mutable token lists may store the information as well for testing
224      * purposes.
225      * </p>
226      *
227      * @param index index of the existing token.
228      * @return lexer's state after recognition of this token
229      * or null for default state. Return null if this token list
230      * does not maintain states.
231      */

232     Object JavaDoc state(int index);
233     
234     /**
235      * Returns true if the underlying token list does not contain offset ranges
236      * that would not be covered by tokens.
237      * <br/>
238      * This could happen if a batch token list would use token id filter.
239      * <br/>
240      * If the token list is continuous the TokenSequence
241      * can compute token offsets more efficiently.
242      */

243     boolean isContinuous();
244
245     /**
246      * Get set of token ids to be skipped during token creation.
247      */

248     Set JavaDoc<T> skipTokenIds();
249
250 }
251
Popular Tags