KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > netbeans > modules > ruby > lexer > RubyTokenId


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */
19 package org.netbeans.modules.ruby.lexer;
20
21 import java.util.ArrayList JavaDoc;
22 import java.util.Collection JavaDoc;
23 import java.util.HashMap JavaDoc;
24 import java.util.List JavaDoc;
25 import java.util.Map JavaDoc;
26 import org.netbeans.api.gsf.GsfTokenId;
27 import org.netbeans.api.lexer.InputAttributes;
28 import org.netbeans.api.lexer.Language;
29 import org.netbeans.api.lexer.LanguagePath;
30 import org.netbeans.api.lexer.Token;
31 import org.netbeans.api.lexer.TokenId;
32 import org.netbeans.api.lexer.TokenId;
33 import org.netbeans.modules.ruby.RubyMimeResolver;
34 import org.netbeans.modules.ruby.lexer.RubyLexer;
35 import org.netbeans.modules.ruby.lexer.RubyStringTokenId;
36 import org.netbeans.spi.lexer.LanguageEmbedding;
37 import org.netbeans.spi.lexer.LanguageHierarchy;
38 import org.netbeans.spi.lexer.Lexer;
39 import org.netbeans.spi.lexer.LexerRestartInfo;
40
41
42 /**
43  *
44  * @author Tor Norbye
45  */
46 public class RubyTokenId extends GsfTokenId {
47     // FixedText on these tokens is risky since the JRuby parser in some cases
48 // seems to use these tokens for multiple source words; when I try opening
49 // for example imap.rb Bad Stuff happens. For now compute token text on the fly.
50 public static final GsfTokenId DOT = new GsfTokenId("DOT", null, "operator");
51     public static final GsfTokenId RANGE = new GsfTokenId("RANGE", null, "operator");
52     public static final GsfTokenId COLON3 = new GsfTokenId("COLON3", null, "operator");
53     public static final GsfTokenId SUPER = new GsfTokenId("SUPER", null, "keyword");
54     public static final GsfTokenId SELF = new GsfTokenId("SELF", null, "keyword");
55     public static final GsfTokenId QUOTED_STRING_LITERAL =
56         new GsfTokenId("QUOTED_STRING_LITERAL", null, "string");
57     public static final GsfTokenId QUOTED_STRING_BEGIN =
58         new GsfTokenId("QUOTED_STRING_BEGIN", null, "string");
59     public static final GsfTokenId QUOTED_STRING_END =
60         new GsfTokenId("QUOTED_STRING_END", null, "string");
61     public static final GsfTokenId EMBEDDED_RUBY =
62         //new GsfTokenId("EMBEDDED_RUBY", null, "string");
63 new GsfTokenId("EMBEDDED_RUBY", null, "default");
64     public static final GsfTokenId MODULE = new GsfTokenId("MODULE", null, "keyword");
65     public static final GsfTokenId CLASS = new GsfTokenId("CLASS", null, "keyword");
66     public static final GsfTokenId DEF = new GsfTokenId("DEF", null, "keyword");
67     public static final GsfTokenId END = new GsfTokenId("END", null, "keyword");
68     public static final GsfTokenId BEGIN = new GsfTokenId("BEGIN", null, "keyword");
69     public static final GsfTokenId FOR = new GsfTokenId("FOR", null, "keyword");
70     public static final GsfTokenId CASE = new GsfTokenId("CASE", null, "keyword");
71     public static final GsfTokenId LOOP = new GsfTokenId("LOOP", null, "keyword");
72
73     /** The "do" keyword, when used as a block (NOT as a statement modifier or in a block/closure) */
74     public static final GsfTokenId DO = new GsfTokenId("DO", null, "keyword");
75
76     /** The "if" keyword, when used as a block (NOT as a statement modifier) */
77     public static final GsfTokenId IF = new GsfTokenId("IF", null, "keyword");
78
79     /** The "while" keyword, when used as a block (NOT as a statement modifier) */
80     public static final GsfTokenId WHILE = new GsfTokenId("WHILE", null, "keyword");
81
82     /** The "until" keyword, when used as a block (NOT as a statement modifier) */
83     public static final GsfTokenId UNTIL = new GsfTokenId("UNTIL", null, "keyword");
84
85     /** The "unless" keyword, when used as a block (NOT as a statement modifier) */
86     public static final GsfTokenId UNLESS = new GsfTokenId("UNLESS", null, "keyword");
87
88     // Indent words
89 public static final GsfTokenId ELSE = new GsfTokenId("ELSE", null, "keyword");
90     public static final GsfTokenId ELSIF = new GsfTokenId("ELSIF", null, "keyword");
91     public static final GsfTokenId ENSURE = new GsfTokenId("ENSURE", null, "keyword");
92     public static final GsfTokenId WHEN = new GsfTokenId("WHEN", null, "keyword");
93     public static final GsfTokenId RESCUE = new GsfTokenId("RESCUE", null, "keyword");
94     private static final Language<GsfTokenId> language =
95         new LanguageHierarchy<GsfTokenId>() {
96                 protected String JavaDoc mimeType() {
97                     return RubyMimeResolver.RUBY_MIME_TYPE;
98                 }
99
100                 protected Collection JavaDoc<GsfTokenId> createTokenIds() {
101                     return getUsedTokens();
102                 }
103
104                 protected Map JavaDoc<String JavaDoc, Collection JavaDoc<GsfTokenId>> createTokenCategories() {
105                     Map JavaDoc<String JavaDoc, Collection JavaDoc<GsfTokenId>> cats =
106                         new HashMap JavaDoc<String JavaDoc, Collection JavaDoc<GsfTokenId>>();
107
108                     // // Incomplete tokens
109 // cats.put("incomplete", EnumSet.of(
110 // RubyTokenId.CHAR_LITERAL_INCOMPLETE,
111 // RubyTokenId.STRING_LITERAL_INCOMPLETE,
112 // RubyTokenId.BLOCK_COMMENT_INCOMPLETE
113 // ));
114 // // Additional literals being a lexical error
115 // cats.put("error", EnumSet.of(
116 // RubyTokenId.CHAR_LITERAL_INCOMPLETE,
117 // RubyTokenId.STRING_LITERAL_INCOMPLETE,
118 // RubyTokenId.BLOCK_COMMENT_INCOMPLETE,
119 // RubyTokenId.FLOAT_LITERAL_INVALID
120 // ));
121 // // Complete and incomplete literals
122 // EnumSet<RubyTokenId> l = EnumSet.of(
123 // RubyTokenId.INT_LITERAL,
124 // RubyTokenId.LONG_LITERAL,
125 // RubyTokenId.FLOAT_LITERAL,
126 // RubyTokenId.DOUBLE_LITERAL,
127 // RubyTokenId.CHAR_LITERAL
128 // );
129 // l.addAll(EnumSet.of(
130 // RubyTokenId.CHAR_LITERAL_INCOMPLETE,
131 // RubyTokenId.STRING_LITERAL,
132 // RubyTokenId.STRING_LITERAL_INCOMPLETE
133 // ));
134 // cats.put("literal", l);
135 return cats;
136                 }
137
138                 protected Lexer<GsfTokenId> createLexer(LexerRestartInfo<GsfTokenId> info) {
139                     return RubyLexer.create(info);
140                 }
141
142                 @Override JavaDoc
143                 protected LanguageEmbedding<?extends TokenId> embedding(Token<GsfTokenId> token,
144                     LanguagePath languagePath, InputAttributes inputAttributes) {
145                     GsfTokenId id = token.id();
146
147                     if (id == QUOTED_STRING_LITERAL) {
148                         return LanguageEmbedding.create(RubyStringTokenId.languageDouble(), 0, 0);
149                     } else if (id == STRING_LITERAL) {
150                         return LanguageEmbedding.create(RubyStringTokenId.languageSingle(), 0, 0);
151                     } else if ((id == LINE_COMMENT) || (id == BLOCK_COMMENT) ||
152                             (id == BLOCK_COMMENT_INCOMPLETE)) {
153                         return LanguageEmbedding.create(RubyCommentTokenId.language(), 1, 0);
154
155                         //} else if (id == REGEXP_LITERAL) {
156 // return LanguageEmbedding.create(RubyRegexpTokenId.language(), 0, 0);
157 } else if (id == EMBEDDED_RUBY) {
158                         return LanguageEmbedding.create(RubyTokenId.language(), 0, 0);
159                     }
160
161                     return null; // No embedding
162 }
163
164                 // protected CharPreprocessor createCharPreprocessor() {
165 // return CharPreprocessor.createUnicodeEscapesPreprocessor();
166 // }
167 }.language();
168
169     public RubyTokenId(String JavaDoc name, String JavaDoc fixedText, String JavaDoc primaryCategory) {
170         super(name, fixedText, primaryCategory);
171     }
172
173     public static List JavaDoc<GsfTokenId> getUsedTokens() {
174         List JavaDoc<GsfTokenId> types = new ArrayList JavaDoc<GsfTokenId>(20);
175         types.add(ERROR);
176         types.add(IDENTIFIER);
177         types.add(CLASS_VAR);
178         types.add(INSTANCE_VAR);
179         types.add(GLOBAL_VAR);
180         types.add(CONSTANT);
181         types.add(DOCUMENTATION);
182         types.add(INT_LITERAL);
183         types.add(REGEXP_LITERAL);
184         types.add(LONG_LITERAL);
185         types.add(FLOAT_LITERAL);
186         types.add(DOUBLE_LITERAL);
187         types.add(CHAR_LITERAL);
188         types.add(QUOTED_STRING_LITERAL);
189         types.add(STRING_LITERAL);
190         types.add(WHITESPACE);
191         types.add(LINE_COMMENT);
192         types.add(BLOCK_COMMENT);
193         types.add(TODO);
194         types.add(TYPE_SYMBOL);
195         types.add(EMBEDDED_RUBY);
196
197         types.add(LPAREN);
198         types.add(RPAREN);
199         types.add(LBRACE);
200         types.add(RBRACE);
201         types.add(LBRACKET);
202         types.add(RBRACKET);
203         types.add(STRING_BEGIN);
204         types.add(STRING_END);
205         types.add(REGEXP_BEGIN);
206         types.add(REGEXP_END);
207
208         // Keywords: block oriented
209 types.add(DEF);
210         types.add(END);
211         types.add(DO);
212         types.add(BEGIN);
213         types.add(IF);
214         types.add(CLASS);
215         types.add(MODULE);
216         types.add(FOR);
217         types.add(CASE);
218         types.add(LOOP);
219         types.add(WHILE);
220         types.add(UNTIL);
221         types.add(UNLESS);
222
223         types.add(ANY_KEYWORD);
224         types.add(ANY_OPERATOR);
225
226         return types;
227     }
228
229     public static Language<GsfTokenId> language() {
230         return language;
231     }
232 }
233
Popular Tags