LexerUtilsConstants


1   /*
2    * The contents of this file are subject to the terms of the Common Development
3    * and Distribution License (the License). You may not use this file except in
4    * compliance with the License.
5    *
6    * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7    * or http://www.netbeans.org/cddl.txt.
8    *
9    * When distributing Covered Code, include this CDDL Header Notice in each file
10   * and include the License file at http://www.netbeans.org/cddl.txt.
11   * If applicable, add the following below the CDDL Header, with the fields
12   * enclosed by brackets [] replaced by your own identifying information:
13   * "Portions Copyrighted [year] [name of copyright owner]"
14   *
15   * The Original Software is NetBeans. The Initial Developer of the Original
16   * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17   * Microsystems, Inc. All Rights Reserved.
18   */
19  
20  package org.netbeans.lib.lexer;
21  
22  import org.netbeans.api.lexer.InputAttributes;
23  import org.netbeans.api.lexer.Language;
24  import org.netbeans.api.lexer.LanguagePath;
25  import org.netbeans.api.lexer.Token;
26  import org.netbeans.api.lexer.TokenHierarchy;
27  import org.netbeans.api.lexer.TokenId;
28  import org.netbeans.lib.editor.util.ArrayUtilities;
29  import org.netbeans.lib.lexer.inc.SnapshotTokenList;
30  import org.netbeans.spi.lexer.LanguageHierarchy;
31  import org.netbeans.spi.lexer.LexerInput;
32  import org.netbeans.lib.lexer.token.AbstractToken;
33  import org.netbeans.spi.lexer.LanguageEmbedding;
34  import org.netbeans.spi.lexer.TokenPropertyProvider;
35  
36  /**
37   * Various utility methods and constants in lexer module.
38   *
39   * @author Miloslav Metelka
40   * @version 1.00
41   */
42  
43  public final class LexerUtilsConstants {
44      
45      /**
46       * Maximum allowed number of consecutive flyweight tokens.
47       * <br>
48       * High number of consecutive flyweight tokens
49       * would degrade performance of offset
50       * finding.
51       */
52      public static final int MAX_FLY_SEQUENCE_LENGTH = 5;
53      
54      /**
55       * Minimum number of characters that will be lexed
56       * at once in a mutable input setup.
57       * <br>
58       * The created tokens will be notified in one token change event.
59       * <br>
60       * This should roughly cover a single page with text
61       * (so that an initial page of text is lexed at once)
62       * but it's not strictly necessary.
63       */
64      public static final int MIN_LEXED_AREA_LENGTH = 4096;
65      
66      /**
67       * Fraction of the mutable input size that will be lexed at once.
68       * <br>
69       * This should avoid notifying of token creations too many times
70       * for large inputs.
71       */
72      public static final int LEXED_AREA_INPUT_SIZE_FRACTION = 10;
73      
74      /**
75       * Check that there are no more characters to be read from the given
76       * lexer input operation.
77       */
78      public static void checkLexerInputFinished(CharProvider input, LexerInputOperation operation) {
79          if (input.read() != LexerInput.EOF) {
80              throw new IllegalStateException  (
81                  "Lexer " + operation.lexer() + // NOI18N
82                  " returned null token" + // NOI18N
83                  " but EOF was not read from lexer input yet." + // NOI18N
84                  " Fix the lexer."// NOI18N
85              );
86          }
87          if (input.readIndex() > 0) {
88              throw new IllegalStateException  (
89                  "Lexer " + operation.lexer() + // NOI18N
90                  " returned null token but lexerInput.readLength()=" + // NOI18N
91                  input.readIndex() +
92                  " - these characters need to be tokenized." + // NOI18N
93                  " Fix the lexer." // NOI18N
94              );
95          }
96      }
97      
98      public static void tokenLengthZeroOrNegative(int tokenLength) {
99          if (tokenLength == 0) {
100             throw new IllegalArgumentException  (
101                 "Tokens with zero length are not supported by the framework." // NOI18N
102               + " Fix the lexer." // NOI18N
103             );
104         } else { // tokenLength < 0
105             throw new IllegalArgumentException  (
106                 "Negative token length " + tokenLength // NOI18N
107             );
108         }
109     }
110 
111     public static void throwFlyTokenProhibited() {
112         throw new IllegalStateException  ("Flyweight token created but prohibited." // NOI18N
113                 + " Lexer needs to check lexerInput.isFlyTokenAllowed()."); // NOI18N
114     }
115 
116     public static void throwBranchTokenFlyProhibited(AbstractToken token) {
117         throw new IllegalStateException  ("Language embedding cannot be created" // NOI18N
118                 + " for flyweight token=" + token // NOI18N
119                 + "\nFix the lexer to not create flyweight token instance when"
120                 + " language embedding exists for the token."
121         );
122     }
123     
124     public static void checkValidBackup(int count, int maxCount) {
125         if (count > maxCount) {
126             throw new IndexOutOfBoundsException  ("Cannot backup " // NOI18N
127                     + count + " characters. Maximum: " // NOI18N
128                     + maxCount + '.');
129         }
130     }
131     
132     public static <T extends TokenId> LanguageHierarchy<T> languageHierarchy(Language<T> language) {
133         return LexerApiPackageAccessor.get().languageHierarchy(language);
134     }
135 
136     public static <T extends TokenId> LanguageOperation<T> languageOperation(Language<T> language) {
137         return LexerSpiPackageAccessor.get().operation(languageHierarchy(language));
138     }
139 
140     /**
141      * Returns the most embedded language in the given language path.
142      * <br/>
143      * The method casts the resulting language to the generic type requested by the caller.
144      */
145     public static <T extends TokenId> Language<T> mostEmbeddedLanguage(LanguagePath languagePath) {
146         @SuppressWarnings  ("unchecked")
147         Language<T> l = (Language<T>)languagePath.innerLanguage();
148         return l;
149     }
150     
151     /**
152      * Returns language hierarchy of the most embedded language in the given language path.
153      * <br/>
154      * The method casts the resulting language hierarchy to the generic type requested by the caller.
155      */
156     public static <T extends TokenId> LanguageHierarchy<T> mostEmbeddedLanguageHierarchy(LanguagePath languagePath) {
157         @SuppressWarnings  ("unchecked")
158         LanguageHierarchy<T> lh = (LanguageHierarchy<T>)languageHierarchy(languagePath.innerLanguage());
159         return lh;
160     }
161     
162     /**
163      * Returns language operation of the most embedded language in the given language path.
164      * <br/>
165      * The method casts the resulting language operation to the generic type requested by the caller.
166      */
167     public static <T extends TokenId> LanguageOperation<T> mostEmbeddedLanguageOperation(LanguagePath languagePath) {
168         @SuppressWarnings  ("unchecked")
169         LanguageOperation<T> lo = (LanguageOperation<T>)LexerSpiPackageAccessor.get().operation(
170                 mostEmbeddedLanguageHierarchy(languagePath));
171         return lo;
172     }
173     
174     /**
175      * Find the language embedding for the given parameters.
176      * <br/>
177      * First the <code>LanguageHierarchy.embedding()</code> method is queried
178      * and if no embedding is found then the <code>LanguageProvider.findLanguageEmbedding()</code>.
179      */
180     public static <T extends TokenId> LanguageEmbedding<? extends TokenId>
181     findEmbedding(Token<T> token, LanguagePath languagePath, InputAttributes inputAttributes) {
182         LanguageHierarchy<T> languageHierarchy = mostEmbeddedLanguageHierarchy(languagePath);
183         LanguageEmbedding<? extends TokenId> embedding =
184                 LexerSpiPackageAccessor.get().embedding(
185                 languageHierarchy, token, languagePath, inputAttributes);
186 
187         if (embedding == null) {
188             // try language embeddings registered in Lookup
189             embedding = LanguageManager.getInstance().findLanguageEmbedding(
190                     token, languagePath, inputAttributes);
191         }
192         return embedding;
193     }
194 
195     /**
196      * Returns token from the given object which is either the token
197      * or an embedding container.
198      * <br/>
199      * The method casts the resulting token to the generic type requested by the caller.
200      */
201     public static <T extends TokenId> AbstractToken<T> token(Object   tokenOrEmbeddingContainer) {
202         @SuppressWarnings  ("unchecked")
203         AbstractToken<T> token = (AbstractToken<T>)
204             ((tokenOrEmbeddingContainer.getClass() == EmbeddingContainer.class)
205                 ? ((EmbeddingContainer)tokenOrEmbeddingContainer).token()
206                 : (AbstractToken<? extends TokenId>)tokenOrEmbeddingContainer);
207         return token;
208     }
209 
210     public static <T extends TokenId> AbstractToken<T> token(TokenList<T> tokenList, int index) {
211         return token(tokenList.tokenOrEmbeddingContainer(index));
212     }
213 
214     public static <T extends TokenId> StringBuilder   appendTokenList(StringBuilder   sb,
215     TokenList<T> tokenList, int currentIndex) {
216         if (sb == null) {
217             sb = new StringBuilder  ();
218         }
219         TokenHierarchy<?> tokenHierarchy;
220         if (tokenList instanceof SnapshotTokenList) {
221                 tokenHierarchy = ((SnapshotTokenList<T>)tokenList).snapshot().tokenHierarchy();
222                 sb.append(tokenList).append('\n');
223         } else {
224                 tokenHierarchy = null;
225         }
226 
227         int tokenCount = tokenList.tokenCountCurrent();
228         int digitCount = ArrayUtilities.digitCount(tokenCount);
229         for (int i = 0; i < tokenCount; i++) {
230             sb.append((i == currentIndex) ? '*' : ' ');
231             ArrayUtilities.appendBracketedIndex(sb, i, digitCount);
232             Object   tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(i);
233             if (tokenOrEmbeddingContainer == null) {
234                 System.err.println("tokenList=" + tokenList + ", i=" + i);
235             }
236             sb.append((tokenOrEmbeddingContainer.getClass() == EmbeddingContainer.class) ? '<' : ' ');
237             sb.append(": ");
238             AbstractToken token = token(tokenOrEmbeddingContainer);
239             sb.append(token.dumpInfo(tokenHierarchy));
240             int la = tokenList.lookahead(i);
241             if (la != 0) {
242                 sb.append(", la=");
243                 sb.append(la);
244             }
245             Object   state= tokenList.state(i);
246             if (state != null) {
247                 sb.append(", s=");
248                 sb.append(state);
249             }
250             sb.append('\n');
251         }
252         return sb;
253     }
254     
255     public static boolean statesEqual(Object   state1, Object   state2) {
256         return (state1 == null && state2 == null)
257             || (state1 != null && state1.equals(state2));
258     }
259     
260     public static String   idToString(TokenId id) {
261         return id.name() + '[' + id.ordinal() + ']'; // NOI18N;
262     }
263     
264     private LexerUtilsConstants() {
265         // no instances
266     }
267 
268 }
269
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags