TokenFactory


1   /*
2    * The contents of this file are subject to the terms of the Common Development
3    * and Distribution License (the License). You may not use this file except in
4    * compliance with the License.
5    *
6    * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7    * or http://www.netbeans.org/cddl.txt.
8    *
9    * When distributing Covered Code, include this CDDL Header Notice in each file
10   * and include the License file at http://www.netbeans.org/cddl.txt.
11   * If applicable, add the following below the CDDL Header, with the fields
12   * enclosed by brackets [] replaced by your own identifying information:
13   * "Portions Copyrighted [year] [name of copyright owner]"
14   *
15   * The Original Software is NetBeans. The Initial Developer of the Original
16   * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17   * Microsystems, Inc. All Rights Reserved.
18   */
19  
20  package org.netbeans.spi.lexer;
21  
22  import java.util.Set  ;
23  import org.netbeans.api.lexer.PartType;
24  import org.netbeans.api.lexer.Token;
25  import org.netbeans.api.lexer.TokenId;
26  import org.netbeans.lib.editor.util.CharSequenceUtilities;
27  import org.netbeans.lib.lexer.LanguageOperation;
28  import org.netbeans.lib.lexer.LexerInputOperation;
29  import org.netbeans.lib.lexer.TokenIdImpl;
30  import org.netbeans.lib.lexer.token.CustomTextToken;
31  import org.netbeans.lib.lexer.token.DefaultToken;
32  import org.netbeans.lib.lexer.token.ComplexToken;
33  import org.netbeans.lib.lexer.token.PreprocessedTextToken;
34  import org.netbeans.lib.lexer.token.ComplexToken;
35  import org.netbeans.lib.lexer.token.PropertyToken;
36  import org.netbeans.lib.lexer.token.TextToken;
37  
38  /**
39   * Lexer should delegate all the token instances creation to this class.
40   * <br/>
41   * It's not allowed to create empty tokens.
42   *
43   * @author Miloslav Metelka
44   * @version 1.00
45   */
46  
47  public final class TokenFactory<T extends TokenId> {
48  
49      /** Flag for additional correctness checks (may degrade performance). */
50      private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");
51  
52      /**
53       * Token instance that should be returned by the lexer
54       * if there is an active filtering of certain token ids
55       * and the just recognized token-id should be skipped.
56       */
57      public static final Token SKIP_TOKEN
58          = new TextToken<TokenId>(
59              new TokenIdImpl("skip-token-id(should never be part of token sequence", 0, null), // NOI18N
60              "" // empty skip token text NOI18N
61          );
62      
63      private final LexerInputOperation<T> operation;
64      
65      TokenFactory(LexerInputOperation<T> operation) {
66          this.operation = operation;
67      }
68  
69      /**
70       * Create token with token length corresponding
71       * to the number of characters read from the lexer input.
72       *
73       * @see #createToken(TokenId, int)
74       */
75      public Token<T> createToken(T id) {
76          return createToken(id, operation.readIndex());
77      }
78  
79      /**
80       * Create regular token instance with an explicit length.
81       *
82       * @param id non-null token id recognized by the lexer.
83       * @param length >=0 length of the token to be created. The length must not
84       *  exceed the number of characters read from the lexer input.
85       * @return non-null regular token instance.
86       *  <br/>
87       *  If there were any characters preprocessed by {@link CharPreprocessor}
88       *  then a special token instance will be created for it.
89       *  <br/>
90       *  {@link #SKIP_TOKEN} will be returned
91       *  if tokens for the given token id should be skipped
92       *  because of token id filter.
93       */
94      public Token<T> createToken(T id, int length) {
95          if (isSkipToken(id)) {
96              operation.tokenRecognized(length, true);
97              return skipToken();
98          } else { // Do not skip the token
99              if (operation.tokenRecognized(length, false)) { // Create preprocessed token
100                 return new PreprocessedTextToken<T>(id, operation.tokenLength());
101             } else {
102                 return new DefaultToken<T>(id, operation.tokenLength());
103             }
104         }
105     }
106 
107     /**
108      * Create regular token instance with an explicit length and part type.
109      *
110      * @param id non-null token id recognized by the lexer.
111      * @param length >=0 length of the token to be created. The length must not
112      *  exceed the number of characters read from the lexer input.
113      * @param partType whether this token is complete token or a part of a complete token.
114      * @return non-null regular token instance.
115      *  <br/>
116      *  If there were any characters preprocessed by {@link CharPreprocessor}
117      *  then a special token instance will be created for it.
118      *  <br/>
119      *  {@link #SKIP_TOKEN} will be returned
120      *  if tokens for the given token id should be skipped
121      *  because of token id filter.
122      */
123     public Token<T> createToken(T id, int length, PartType partType) {
124         checkPartTypeNonNull(partType);
125         if (partType == PartType.COMPLETE)
126             return createToken(id, length);
127 
128         if (isSkipToken(id)) {
129             operation.tokenRecognized(length, true);
130             return skipToken();
131         } else { // Do not skip the token
132             if (operation.tokenRecognized(length, false)) { // Create preprocessed token
133                 return new ComplexToken<T>(id, operation.tokenLength(), null, null, partType);
134             } else {
135                 return new PropertyToken<T>(id, operation.tokenLength(), null, partType);
136             }
137         }
138     }
139 
140     /**
141      * Get flyweight token for the given arguments.
142      * <br/>
143      * <b>Note:</b> The returned token will not be flyweight under certain
144      * conditions - see return value description.
145      *
146      * @param id non-null token id.
147      * @param text non-null text that the flyweight token should carry.
148      * @return non-null flyweight token instance.
149      *  <br/>
150      *  If there were any characters preprocessed by {@link CharPreprocessor}
151      *  then a special non-flyweight token instance will be created for it.
152      *  <br/>
153      *  For performance reasons there is a limit for number of successive
154      *  flyweight tokens. If this limit would be exceeded a single non-flyweight
155      *  token gets created instead of flyweight one.
156      *  <br/>
157      *  {@link #SKIP_TOKEN} will be returned
158      *  if tokens for the given token id should be skipped
159      *  because of token id filter.
160      */
161     public Token<T> getFlyweightToken(T id, String   text) {
162         assert (text.length() <= operation.readIndex());
163         // Compare each recognized char with the corresponding char in text
164         if (testing) {
165             for (int i = 0; i < text.length(); i++) {
166                 if (text.charAt(i) != operation.readExisting(i)) {
167                     throw new IllegalArgumentException  ("Flyweight text in " + // NOI18N
168                             "TokenFactory.getFlyweightToken(" + id + ", \"" + // NOI18N
169                             CharSequenceUtilities.debugText(text) + "\") " + // NOI18N
170                             "differs from recognized text: '" + // NOI18N
171                             CharSequenceUtilities.debugChar(operation.readExisting(i)) +
172                             "' != '" + CharSequenceUtilities.debugChar(text.charAt(i)) + // NOI18N
173                             "' at index=" + i // NOI18N
174                     );
175                 }
176             }
177         }
178 
179         // Check whether token with given id should be created
180         if (isSkipToken(id)) {
181             operation.tokenRecognized(text.length(), true);
182             return skipToken();
183         } else { // Do not skip the token
184             if (operation.tokenRecognized(text.length(), false)) { // Create preprocessed token
185                 return new PreprocessedTextToken<T>(id, operation.tokenLength());
186             } else if (operation.isFlyTokenAllowed()) {
187                 LanguageOperation<T> langOp = operation.languageOperation();
188                 return langOp.getFlyweightToken(id, text);
189             } else { // return non-flyweight token
190                 return new DefaultToken<T>(id, operation.tokenLength());
191             }
192         }
193     }
194     
195     /**
196      * Create token with properties.
197      *
198      * @param id non-null token id.
199      * @param length >=0 length of the token to be created. The length must not
200      *  exceed the number of characters read from the lexer input.
201      * @param propertyProvider non-null token property provider.
202      * @param partType whether this token is complete or just a part of complete token.
203      *  See {@link TokenPropertyProvider} for examples how this parameter may be used.
204      * @return non-null property token instance.
205      *  <br/>
206      *  If there were any characters preprocessed by {@link CharPreprocessor}
207      *  then a special token instance will be created for it.
208      *  <br/>
209      *  {@link #SKIP_TOKEN} will be returned
210      *  if tokens for the given token id should be skipped
211      *  because of token id filter.
212      */
213     public Token<T> createPropertyToken(T id, int length,
214     TokenPropertyProvider propertyProvider, PartType partType) {
215         checkPartTypeNonNull(partType);
216         if (isSkipToken(id)) {
217             operation.tokenRecognized(length, true);
218             return skipToken();
219         } else { // Do not skip the token
220             if (operation.tokenRecognized(length, false)) { // Create preprocessed token
221                 return new ComplexToken<T>(id, operation.tokenLength(),
222                     propertyProvider, null, partType);
223             } else {
224                 return new PropertyToken<T>(id, operation.tokenLength(),
225                     propertyProvider, partType);
226             }
227         }
228     }
229 
230     /**
231      * Create token with a custom text that possibly differs from the text
232      * represented by the token in the input text.
233      */
234     public Token<T> createCustomTextToken(T id, CharSequence   text, int length, PartType partType) {
235         checkPartTypeNonNull(partType);
236         if (isSkipToken(id)) {
237             operation.tokenRecognized(length, true);
238             return skipToken();
239         } else { // Do not skip the token
240             if (operation.tokenRecognized(length, false)) { // Create preprocessed token
241                 return new ComplexToken<T>(id, operation.tokenLength(), null, text, partType);
242             } else {
243                 return new CustomTextToken<T>(id, operation.tokenLength(), text, partType);
244             }
245         }
246     }
247     
248     private boolean isSkipToken(T id) {
249         Set  <? extends TokenId> skipTokenIds = operation.skipTokenIds();
250         return (skipTokenIds != null) && skipTokenIds.contains(id);
251     }
252 
253     @SuppressWarnings  ("unchecked") // NOI18N
254     private Token<T> skipToken() {
255         return SKIP_TOKEN;
256     }
257     
258     private void checkPartTypeNonNull(PartType partType) {
259         if (partType == null)
260             throw new IllegalArgumentException  ("partType must be non-null");
261     }
262     
263 }
264
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags