Java > Open Source Codes > org > netbeans > api > lexer > Language


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */
19
20 package org.netbeans.api.lexer;
21
22 import java.util.ArrayList ;
23 import java.util.Collection ;
24 import java.util.Collections ;
25 import java.util.EnumSet ;
26 import java.util.Map ;
27 import java.util.HashMap ;
28 import java.util.List ;
29 import java.util.Set ;
30 import org.netbeans.lib.lexer.LanguageManager;
31 import org.netbeans.lib.lexer.LexerApiPackageAccessor;
32 import org.netbeans.lib.lexer.LexerSpiPackageAccessor;
33 import org.netbeans.lib.lexer.TokenIdSet;
34 import org.netbeans.lib.lexer.TokenHierarchyOperation;
35 import org.netbeans.lib.lexer.inc.TokenChangeInfo;
36 import org.netbeans.lib.lexer.inc.TokenHierarchyEventInfo;
37 import org.netbeans.lib.lexer.inc.TokenListChange;
38 import org.netbeans.spi.lexer.LanguageHierarchy;
39
40 /**
41  * Language describes a set of token ids
42  * that comprise the given language.
43  * <br/>
44  * Each language corresponds to a certain mime-type.
45  * <br/>
46  * An input source may be lexed by using an existing language
47  * - see {@link TokenHierarchy} which is an entry point into the Lexer API.
48  * <br>
49  * Language hierarchy is represented by an unmodifiable set of {@link TokenId}s
50  * that can be retrieved by {@link #tokenIds()} and token categories
51  * {@link #tokenCategories()}.
52  *
53  * <p>
54  * The language cannot be instantiated directly.
55  * <br/>
56  * Instead it should be obtained from {@link LanguageHierarchy#language()}
57  * on an existing language hierarchy.
58  *
59  * @see LanguageHierarchy
60  * @see TokenId
61  *
62  * @author Miloslav Metelka
63  * @version 1.00
64  */
65
66 public final class Language<T extends TokenId> {
67     
68     static {
69         LexerApiPackageAccessor.register(new Accessor());
70     }
71     
72     private final LanguageHierarchy<T> languageHierarchy;
73     
74     private String mimeType;
75     
76     private final int maxOrdinal;
77
78     private final Set <T> ids;
79
80     /** Lazily inited indexed ids for quick translation of ordinal to token id. */
81     private TokenIdSet<T> indexedIds;
82
83     private final Map <String ,T> idName2id;
84     
85     /**
86      * Map of category to ids that it contains.
87      */
88     private final Map <String ,Set <T>> cat2ids;
89     
90     /**
91      * Lists of token categories for particular id.
92      * <br/>
93      * It's a list because it is ordered (primary category is first).
94      */
95     private List <String >[] id2cats;
96     
97     /**
98      * Lists of non-primary token categories for particular id.
99      * <br/>
100      * It's a list because the order might be important
101      * (e.g. for syntax coloring information resolving) although
102      * the present SPI does not utilize that.
103      */
104     private List <String >[] id2nonPrimaryCats;
105     
106     /**
107      * Finds a language by its mime type.
108      *
109      * <p>This method uses information from <code>LanguageProvider</code>s registered
110      * in the default lookup to find <code>Language</code> for a given
111      * mime path.
112      *
113      * <div class="nonnormative">
114      * <p>Netbeans provide an implementation of <code>LanguageProvider</code>
115      * that reads data from the <code>Editors</code> folder on the system filesystem.
116      * Therefore Netbeans modules can register their <code>Language</code>s
117      * in MimeLookup as any other mime-type related service.
118      *
119      * <p>Since this method takes <code>mimePath</code> as a parameter it is
120      * possible to look up <code>Language</code> registered for
121      * a mime type that is embedded in some other mime type (i.e. the nesting
122      * can go any number of levels deep). In reality, however, there is usually
123      * no difference between a language, which uses its own file and the same
124      * language embedded in some other language's file (e.g. there is no difference
125      * between java language in <code>hello.java</code> and a java scriplet in
126      * <code>goodbye.jsp</code>). Therefore <code>Language</code>s are
127      * usually registered only for top level mime types and if a language
128      * can be embedded in another language the embedded language is referenced
129      * by its top level mime type.
130      * </div>
131      *
132      * @param mimePath The mime path of the language you want to find.
133      * @return The <code>Language</code> of the language registered
134      * for the given <code>mimePath</code>.
135      */
136     public static Language<? extends TokenId> find(String mimePath) {
137         return LanguageManager.getInstance().findLanguage(mimePath);
138     }
139     
140     /**
141      * Construct language by providing a collection of token ids
142      * that comprise the language and extra categories into which the token ids belong.
143      *
144      * @param languageHierarchy non-null language hierarchy is in one-to-one relationship
145      * with the language and represents it on SPI side.
146      * @throws IndexOutOfBoundsException if any token id's ordinal is &lt; 0.
147      */
148     Language(LanguageHierarchy<T> languageHierarchy) {
149         this.languageHierarchy = languageHierarchy;
150         mimeType = LexerSpiPackageAccessor.get().mimeType(languageHierarchy);
151         checkMimeTypeValid(mimeType);
152         // Create ids and find max ordinal
153 Collection <T> createdIds = LexerSpiPackageAccessor.get().createTokenIds(languageHierarchy);
154         if (createdIds == null)
155             throw new IllegalArgumentException ("Ids cannot be null"); // NOI18N
156 maxOrdinal = TokenIdSet.findMaxOrdinal(createdIds);
157
158         // Convert collection of ids to efficient indexed Set<T>
159 if (createdIds instanceof EnumSet ) {
160             ids = (Set <T>)createdIds;
161         } else { // not EnumSet
162 ids = new TokenIdSet<T>(createdIds, maxOrdinal, true);
163         }
164         
165         // Create TokenIdSet instances for token categories
166 Map <String ,Collection <T>> createdCat2ids
167                 = LexerSpiPackageAccessor.get().createTokenCategories(languageHierarchy);
168         if (createdCat2ids == null) {
169             createdCat2ids = Collections.emptyMap();
170         }
171         cat2ids = new HashMap <String ,Set <T>>((int)(createdCat2ids.size() / 0.73f));
172         for (Map.Entry <String ,Collection <T>> entry : createdCat2ids.entrySet()) {
173             Collection <T> createdCatIds = entry.getValue();
174             TokenIdSet.checkIdsFromLanguage(createdCatIds, ids);
175             // Do not use the original createdCatIds set because of the following:
176 // 1) Two token categories will have the same sets of contained ids
177 // in the createdCatIds map (the same physical Set instance).
178 // 2) At least one token id will have one of the two categories
179 // as its primary category.
180 // 3) If the original Set instance from the createdCatIds would be used
181 // then both categories would incorrectly contain the extra id(s).
182 Set <T> catIds = new TokenIdSet<T>(createdCatIds, maxOrdinal, false);
183             cat2ids.put(entry.getKey(), catIds);
184         }
185
186         // Walk through all ids and check duplicate names and primary categories
187 idName2id = new HashMap <String ,T>((int)(ids.size() / 0.73f));
188         for (T id : ids) {
189             T sameNameId = idName2id.put(id.name(), id);
190             if (sameNameId != null && sameNameId != id) { // token ids with same name
191 throw new IllegalArgumentException (id +
192                         " has duplicate name with " + sameNameId);
193             }
194
195             String cat = id.primaryCategory();
196             if (cat != null) {
197                 Set <T> catIds = cat2ids.get(cat);
198                 if (catIds == null) {
199                     catIds = new TokenIdSet<T>(null, maxOrdinal, false);
200                     cat2ids.put(cat, catIds);
201                 }
202                 catIds.add(id);
203             }
204         }
205     }
206     
207     /**
208      * Get unmodifiable set of ids contained in this language.
209      * <br/>
210      * An iterator over the set returns the ids sorted by their ordinals.
211      *
212      * @return unmodifiable set of ids contained in this language.
213      */
214     public Set <T> tokenIds() {
215         return ids;
216     }
217     
218     /**
219      * Get tokenId for the given ordinal. This method
220      * can be used by lexers to quickly translate ordinal
221      * to tokenId.
222      * @param ordinal ordinal to be translated to corresponding tokenId.
223      * @return valid tokenId or null if there's no corresponding
224      * tokenId for the given int-id. It's possible because intIds
225      * of the language's token ids do not need to be continuous.
226      * If the ordinal is &lt;0 or higher than the highest
227      * ordinal of all the token ids of this language the method
228      * throws {@link IndexOutOfBoundsException}.
229      * @throws IndexOutOfBoundsException if the ordinal is
230      * &lt;0 or higher than {@link #maxOrdinal()}.
231      */
232     public T tokenId(int ordinal) {
233         synchronized (idName2id) {
234             if (indexedIds == null) {
235                 if (ids instanceof EnumSet ) {
236                     indexedIds = new TokenIdSet<T>(ids, maxOrdinal, false);
237                 } else { // not EnumSet
238 indexedIds = (TokenIdSet<T>)ids;
239                 }
240             }
241             return indexedIds.indexedIds()[ordinal];
242         }
243     }
244     
245     /**
246      * Similar to {@link #tokenId(int)} however it guarantees
247      * that it will always return non-null tokenId. Typically for a lexer
248      * just being developed it's possible that there are some integer
249      * token ids defined in the generated lexer for which there is
250      * no correspondence in the language. The lexer wrapper should
251      * always call this method if it expects to find a valid
252      * counterpart for given integer id.
253      * @param ordinal ordinal to translate to token id.
254      * @return always non-null tokenId that corresponds to the given integer id.
255      * @throws IndexOutOfBoundsException if the ordinal is
256      * &lt;0 or higher than {@link #maxOrdinal()} or when there is no corresponding
257      * token id for it.
258      */
259     public T validTokenId(int ordinal) {
260         T id = tokenId(ordinal);
261         if (id == null) {
262             throw new IndexOutOfBoundsException ("No tokenId for ordinal=" + ordinal
263                 + " in language " + this);
264         }
265         return id;
266     }
267     
268     /**
269      * Find the tokenId from its name.
270      * @param name name of the tokenId to find.
271      * @return tokenId with the requested name or null if it does not exist.
272      */
273     public T tokenId(String name) {
274         return idName2id.get(name);
275     }
276     
277     /**
278      * Similar to {@link #tokenId(String)} but guarantees a valid tokenId to be returned.
279      * @throws IllegalArgumentException if no token in this language has the given name.
280      */
281     public T validTokenId(String name) {
282         T id = tokenId(name);
283         if (id == null) {
284             throw new IllegalArgumentException ("No tokenId for name=\"" + name
285                 + "\" in language " + this);
286         }
287         return id;
288     }
289     
290     /**
291      * Get maximum ordinal of all the token ids that this language contains.
292      * @return maximum integer ordinal of all the token ids that this language contains
293      * or <code>-1</code> if the language contains no token ids.
294      */
295     public int maxOrdinal() {
296         return maxOrdinal;
297     }
298
299     /**
300      * Get names of all token categories of this language.
301      *
302      * @return unmodifiable set containing names of all token categories
303      * contained in this language.
304      */
305     public Set <String > tokenCategories() {
306         return Collections.unmodifiableSet(cat2ids.keySet());
307     }
308
309     /**
310      * Get members of the category with given name.
311      *
312      * @param tokenCategory non-null name of the category.
313      * @return set of token ids belonging to the given category.
314      */
315     public Set <T> tokenCategoryMembers(String tokenCategory) {
316         return Collections.unmodifiableSet(cat2ids.get(tokenCategory));
317     }
318     
319     /**
320      * Get list of all token categories for the particular token id.
321      *
322      * @return non-null unmodifiable list of all token categories for the particular token id.
323      * <br>
324      * Primary token's category (if defined for the token id) will be contained
325      * as first one in the list.
326      * @throws IllegalArgumentException if the given token id does not belong
327      * to this language.
328      */
329     public List <String > tokenCategories(T tokenId) {
330         checkMemberId(tokenId);
331         synchronized (idName2id) {
332             if (id2cats == null) {
333                 buildTokenIdCategories();
334             }
335             return id2cats[tokenId.ordinal()];
336         }
337     }
338     
339     /**
340      * Get list of non-primary token categories (not containing the primary category)
341      * for the particular token id.
342      * <br/>
343      * If the token id has no primary category defined then the result
344      * of this method is equal to {@link #tokenCategories(TokenId)}.
345      *
346      * @return non-null unmodifiable list of secondary token categories for the particular token id.
347      * Primary token's category (if defined for the token id) will not be contained
348      * in the list.
349      * @throws IllegalArgumentException if the given token id does not belong
350      * to this language.
351      */
352     public List <String > nonPrimaryTokenCategories(T tokenId) {
353         checkMemberId(tokenId);
354         synchronized (idName2id) {
355             if (id2nonPrimaryCats == null) {
356                 buildTokenIdCategories();
357             }
358             return id2nonPrimaryCats[tokenId.ordinal()];
359         }
360     }
361     
362     /**
363      * Merge two collections of token ids from this language
364      * into an efficient indexed set (the implementation similar
365      * to {@link java.util.EnumSet}).
366      *
367      * @param tokenIds1 non-null collection of token ids to be contained in the returned set.
368      * @param tokenIds2 collection of token ids to be contained in the returned set.
369      * @return set of token ids indexed by their ordinal number.
370      */
371     public Set <T> merge(Collection <T> tokenIds1, Collection <T> tokenIds2) {
372         TokenIdSet.checkIdsFromLanguage(tokenIds1, ids);
373         // Cannot retain EnumSet as tokenIds will already be wrapped
374 // by unmodifiableSet()
375 Set <T> ret = new TokenIdSet<T>(tokenIds1, maxOrdinal, false);
376         if (tokenIds2 != null) {
377             TokenIdSet.checkIdsFromLanguage(tokenIds2, ids);
378             ret.addAll(tokenIds2);
379         }
380         return ret;
381     }
382
383     /**
384      * Gets the mime type of this language.
385      *
386      * @return non-null language's mime type.
387      */
388     public String mimeType() {
389         return mimeType;
390     }
391     
392     /** The languages are equal only if they are the same objects. */
393     public boolean equals(Object obj) {
394         return super.equals(obj);
395     }
396     
397     /** The hashCode of the language is the identity hashCode. */
398     public int hashCode() {
399         return super.hashCode();
400     }
401
402     private void buildTokenIdCategories() {
403         assignCatArrays();
404         // List for decreasing of the number of created maps
405 // for tokenId2category mappings.
406 // List.get(0) is a Map[category, list-of-[category]].
407 // List.get(1) is a Map[category1, Map[category2, list-of-[category1;category2]]].
408 // etc.
409 List <Map <String ,Object >> catMapsList = new ArrayList <Map <String ,Object >>(4);
410         // All categories for a single token id
411 List <String > idCats = new ArrayList <String >(4);
412         for (T id : ids) {
413             // No extra sorting of the categories in which the particular id is contained
414 // - making explicit order of the categories could possibly be acomplished
415 // in the future if necessary by supporting some extra hints
416 // Add all the categories for the particular id into idCats
417 for (Map.Entry <String ,Set <T>> e : cat2ids.entrySet()) {
418                 if (e.getValue().contains(id)) {
419                     idCats.add(e.getKey()); // Add this category to id's categories
420 }
421             }
422             // Assign both non-primary cats and all cats
423 id2cats[id.ordinal()] = findCatList(catMapsList, idCats, 0);
424             id2nonPrimaryCats[id.ordinal()] = findCatList(catMapsList, idCats, 1);
425
426             idCats.clear(); // reuse the list (is cloned if added to catMapsList)
427 }
428     }
429
430     /**
431      * Find the cached list of categories from the catMapsList
432      * for the particular temporarily collected list of categories.
433      *
434      * @param catMapsList non-null list of cached maps.
435      * <br/>
436      * List.get(0) is a Map[category, list-containing-[category]].
437      * <br/>
438      * List.get(1) is a Map[category1, Map[category2, list-containing-[category1;category2]]].
439      * <br/>
440      * etc.
441      *
442      * @param idCats non-null temporarily collected list of categories for the particular id.
443      * It may be modified after this method gets finished.
444      * @param startIndex >=0 starting index in idCats - either 0 for returning
445      * of all categories or 1 for returning non-primary categories.
446      * @return non-null cached list of categories with contents equal to idCats.
447      */
448     private static List <String > findCatList(List <Map <String ,Object >> catMapsList, List <String > idCats, int startIndex) {
449         int size = idCats.size() - startIndex;
450         if (size <= 0) {
451             return Collections.emptyList();
452         }
453         while (catMapsList.size() < size) {
454             catMapsList.add(new HashMap <String ,Object >());
455         }
456         // Find the catList as the last item in the cascaded search through the maps
457 Map <String ,Object > m = catMapsList.get(--size);
458         for (int i = startIndex; i < size; i++) {
459             @SuppressWarnings ("unchecked")
460             Map <String ,Object > catMap = (Map <String ,Object >)m.get(idCats.get(i));
461             if (catMap == null) {
462                 catMap = new HashMap <String ,Object >();
463 // Map<String,Map<String,Object>>
464 m.put(idCats.get(i), catMap);
465             }
466             m = catMap;
467         }
468
469         @SuppressWarnings ("unchecked")
470         List <String > catList = (List <String >)m.get(idCats.get(size));
471         if (catList == null) {
472             catList = new ArrayList <String >(idCats.size() - startIndex);
473             catList.addAll((startIndex > 0)
474                     ? idCats.subList(startIndex, idCats.size())
475                     : idCats);
476             m.put(idCats.get(size), catList);
477         }
478         return catList;
479     }
480         
481     @SuppressWarnings ("unchecked")
482     private void assignCatArrays() {
483         id2cats = (List <String >[])new List [maxOrdinal + 1];
484         id2nonPrimaryCats = (List <String >[])new List [maxOrdinal + 1];
485     }
486
487     /**
488      * Dump list of token ids for this language into string.
489      *
490      * @return dump of contents of this language.
491      */
492     public String dumpInfo() {
493         StringBuilder sb = new StringBuilder ();
494         for (T id : ids) {
495             sb.append(id);
496             List <String > cats = tokenCategories(id);
497             if (cats.size() > 0) {
498                 sb.append(": ");
499                 for (int i = 0; i < cats.size(); i++) {
500                     if (i > 0) {
501                         sb.append(", ");
502                     }
503                     String cat = (String )cats.get(i);
504                     sb.append('"');
505                     sb.append(cat);
506                     sb.append('"');
507                 }
508             }
509         }
510         return ids.toString();
511     }
512     
513     public String toString() {
514         return mimeType + ", LH: " + languageHierarchy;
515     }
516     
517     private void checkMemberId(T id) {
518         if (!ids.contains(id)) {
519             throw new IllegalArgumentException (id + " does not belong to language " + this); // NOI18N
520 }
521     }
522     
523     private static void checkMimeTypeValid(String mimeType) {
524         if (mimeType == null) {
525             throw new IllegalStateException ("mimeType cannot be null"); // NOI18N
526 }
527         int slashIndex = mimeType.indexOf('/');
528         if (slashIndex == -1) { // no slash
529 throw new IllegalStateException ("mimeType=" + mimeType + " does not contain '/'"); // NOI18N
530 }
531         if (mimeType.indexOf('/', slashIndex + 1) != -1) {
532             throw new IllegalStateException ("mimeType=" + mimeType + " contains more than one '/'"); // NOI18N
533 }
534     }
535     
536     /**
537      * Return language hierarchy associated with this language.
538      * <br>
539      * This method is for API package accessor only.
540      */
541     LanguageHierarchy<T> languageHierarchy() {
542         return languageHierarchy;
543     }
544         
545     /**
546      * Accessor of package-private things in this package
547      * that need to be used by the lexer implementation classes.
548      */
549     private static final class Accessor extends LexerApiPackageAccessor {
550
551         public <T extends TokenId> Language<T> createLanguage(
552         LanguageHierarchy<T> languageHierarchy) {
553             return new Language<T>(languageHierarchy);
554         }
555         
556         public <T extends TokenId> LanguageHierarchy<T> languageHierarchy(
557         Language<T> language) {
558             return language.languageHierarchy();
559         }
560         
561         public <I> TokenHierarchy<I> createTokenHierarchy(
562         TokenHierarchyOperation<I,?> tokenHierarchyOperation) {
563             return new TokenHierarchy<I>(tokenHierarchyOperation);
564         }
565         
566         public TokenHierarchyEvent createTokenChangeEvent(
567         TokenHierarchyEventInfo info) {
568             return new TokenHierarchyEvent(info);
569         }
570
571         public <T extends TokenId> TokenChange<T> createTokenChange(
572         TokenChangeInfo<T> info) {
573             return new TokenChange<T>(info);
574         }
575         
576         public <T extends TokenId> TokenChangeInfo<T> tokenChangeInfo(
577         TokenChange<T> tokenChange) {
578             return tokenChange.info();
579         }
580         
581         public <I> TokenHierarchyOperation<I,?> tokenHierarchyOperation(
582         TokenHierarchy<I> tokenHierarchy) {
583             return tokenHierarchy.operation();
584         }
585
586     }
587
588 }
589
590

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags