TokenList


1   /*
2    * The contents of this file are subject to the terms of the Common Development
3    * and Distribution License (the License). You may not use this file except in
4    * compliance with the License.
5    *
6    * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7    * or http://www.netbeans.org/cddl.txt.
8    *
9    * When distributing Covered Code, include this CDDL Header Notice in each file
10   * and include the License file at http://www.netbeans.org/cddl.txt.
11   * If applicable, add the following below the CDDL Header, with the fields
12   * enclosed by brackets [] replaced by your own identifying information:
13   * "Portions Copyrighted [year] [name of copyright owner]"
14   *
15   * The Original Software is NetBeans. The Initial Developer of the Original
16   * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17   * Microsystems, Inc. All Rights Reserved.
18   */
19  
20  package org.netbeans.lib.lexer;
21  
22  import java.util.Set  ;
23  import org.netbeans.api.lexer.LanguagePath;
24  import org.netbeans.api.lexer.InputAttributes;
25  import org.netbeans.api.lexer.TokenId;
26  import org.netbeans.lib.lexer.token.AbstractToken;
27  
28  /**
29   * Browsable list of tokens.
30   * <br>
31   * {@link org.netbeans.api.lexer.TokenSequence} delegates
32   * all its operation to this class so any service provider
33   * delivering this class will be able to produce token sequences.
34   *
35   * There are various implementations of the token list:
36   * <ul>
37   *   <li>BatchTokenList</li> - predecessor of batch token lists
38   *   <li>TextTokenList</li> - token list over immutable char sequence
39   *   <li>CopyTextTokenList</li> - token list over text input
40   *     that needs to be copied. Characters that belong to tokens
41   *     skipped due to skipTokenIds do not need to be copied.
42   *   <li>SkimTokenList</li> - filter over CopyTextTokenList
43   *     to store the token characters in multiple arrays
44   *     and to correctly compute the tokens' starting offsets.
45   *   <li>IncTokenList</li> - token list for mutable-input environment.
46   *   <li>EmbeddedTokenList</li> - token list for a single language embedding
47   *     suitable for both batch and incremental environments.
48   * </ul>
49   *
50   * @author Miloslav Metelka
51   * @version 1.00
52   */
53  
54  public interface TokenList<T extends TokenId> {
55      
56      /**
57       * Language path of this token list.
58       */
59      LanguagePath languagePath();
60      
61      /**
62       * Get token or {@link EmbeddingContainer} at given index in this list.
63       * <br/>
64       * The method's implementation may need to be synchronized as multiple
65       * threads can access it at the same time.
66       * <br/>
67       * The requested index value may be arbitrarily high
68       * (e.g. when TokenSequence.move(index) is used for too high value).
69       *
70       * @param &gt;=0 index of the token in this list.
71       * @return valid token or null if the index is too high.
72       */
73      Object   tokenOrEmbeddingContainer(int index);
74  
75      /**
76       * Replace flyweight token at the given index with its non-flyweight copy.
77       * <br/>
78       * This may be requested by <code>TokenSequence.offsetToken()</code>.
79       *
80       * @param index &gt;=0 index of the flyweight token in this list.
81       * @param flyToken non-null flyweight token. 
82       * @param offset >=0 absolute offset where the flyweight token resides.
83       * @return non-flyweight token instance.
84       */
85      AbstractToken<T> replaceFlyToken(int index, AbstractToken<T> flyToken, int offset);
86      
87      /**
88       * Wrap the token by a branch token list due to language embedding
89       * that exists for the token.
90       *
91       * @param index existing index in this token list at which the token
92       *  should be wrapped with the embedding info.
93       * @param embeddingContainer embedding info that should wrap the token.
94       */
95      void wrapToken(int index, EmbeddingContainer<T> embeddingContainer);
96      
97      /**
98       * Get absolute offset of the token at the given index in the token list.
99       * <br>
100      * This method can only be called if the token at the given index
101      * was already fetched by {@link tokenOrEmbeddingContainer(int)}.
102      * <br/>
103      * For branch token lists this method is only expected to be called
104      * after {@link #updateStartOffsetShift()} was called so it does not perform
105      * any checking whether the start offset of the token list is up-to-date.
106      */
107     int tokenOffset(int index);
108     
109     /**
110      * Get total count of tokens in the list.
111      * <br/>
112      * For token lists that create the tokens lazily
113      * this will lead to lexing till the end of the input.
114      */
115     int tokenCount();
116 
117     /**
118      * Return present number of tokens in the token list but do not create
119      * any new tokens (because of possible lazy token creation).
120      * <br/>
121      * This is necessary e.g. for <code>TokenSequence.move()</code>
122      * that needs a binary search for fast positioning 
123      * but using {@link #tokenCount()} would lead to unnecessary creation
124      * of all tokens.
125      */
126     int tokenCountCurrent();
127 
128     /**
129      * Get number of modifications which mutated this token list.
130      * <br>
131      * Token sequence remembers this number when it gets constructed
132      * and checks this number when it moves between tokens
133      * and if there is an extra modification performed it throws
134      * <code>IllegalStateException</code>.
135      *
136      * <p>
137      * This is also used to check whether this token list corresponds to mutable input
138      * or not because unmodifiable lists return -1 from this method.
139      *
140      * <p>
141      * For branch token lists the {@link #updateStartOffsetShift()} ensures
142      * that the value returned by this method is most up-to-date
143      * (equals to the root list's one).
144      *
145      * @return number of modifications performed to the list.
146      *  <br/>
147      *  Returns -1 if this list is constructed for immutable input and cannot be mutated. 
148      */
149     int modCount();
150     
151     /**
152      * Get absolute offset of the child token with the given raw offset
153      * in the underlying input.
154      *
155      * @param rawOffset raw offset of the child token.
156      * @return absolute offset in the input.
157      */
158     int childTokenOffset(int rawOffset);
159     
160     /**
161      * Get character of a token from the character sequence represented
162      * by this support.
163      *
164      * @param rawOffset raw offset of the child token.
165      *  The given offset value may need to be preprocessed before using (it depends
166      *  on a nature of the token list).
167      * @param index index inside the token's text that should be returned.
168      *  This value cannot be simply added to the previous parameter
169      *  for mutable token lists as the value could errorneously point
170      *  into a middle of the offset gap then.
171      * @return appropriate character that the token has requested.
172      */
173     char childTokenCharAt(int rawOffset, int index);
174 
175     /**
176      * Get the root token list of the token list hierarchy.
177      */
178     TokenList<? extends TokenId> root();
179     
180     /**
181      * Get token hierarchy operation for this token list or null
182      * if this token list does not have any token hierarchy.
183      */
184     TokenHierarchyOperation<?,? extends TokenId> tokenHierarchyOperation();
185     
186     /**
187      * Extra attributes related to the input being lexed.
188      */
189     InputAttributes inputAttributes();
190 
191     /**
192      * Get lookahead information for the token at the existing token index.
193      * <br/>
194      * Lookahead is number of characters that the lexer has read
195      * past the end of the given token in order to recognize it in the text.
196      * <br>
197      * This information allows the lexer to know whether modifications
198      * past the end of the token can affect its validity.
199      *
200      * <p>
201      * In general only mutable token lists benefit from this information
202      * but non-mutable token lists may store the information as well for testing
203      * purposes.
204      * </p>
205      *
206      * @param index index of the existing token.
207      * @return &gt;=0 number of characters that the lexer has read
208      *  in order to recognize this token. Return zero if this token list
209      *  does not maintain lookaheads.
210      */
211     int lookahead(int index);
212     
213     /**
214      * Get state information for the token at the existing token index.
215      * <br/>
216      * It is an object defining lexer's state after recognition
217      * of the given token.
218      * <br/>
219      * This information allows to restart the lexer at the end of the given token.
220      *
221      * <p>
222      * In general only mutable token lists benefit from this information
223      * but non-mutable token lists may store the information as well for testing
224      * purposes.
225      * </p>
226      *
227      * @param index index of the existing token.
228      * @return lexer's state after recognition of this token
229      *  or null for default state. Return null if this token list
230      *  does not maintain states.
231      */
232     Object   state(int index);
233     
234     /**
235      * Returns true if the underlying token list does not contain offset ranges
236      * that would not be covered by tokens.
237      * <br/>
238      * This could happen if a batch token list would use token id filter.
239      * <br/>
240      * If the token list is continuous the TokenSequence
241      * can compute token offsets more efficiently.
242      */
243     boolean isContinuous();
244 
245     /**
246      * Get set of token ids to be skipped during token creation.
247      */
248     Set  <T> skipTokenIds();
249 
250 }
251
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags