KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > jfun > parsec > tokens > Tokenizers


1 package jfun.parsec.tokens;
2
3 import jfun.parsec.Tokenizer;
4
5 /**
6  * The facade class for all pre-built Tokenizer implementations.
7  * <p>
8  * @author Ben Yu
9  * Apr 27, 2006 7:40:16 PM
10  * @since version 0.6
11  */
12 public class Tokenizers {
13
14   /**
15    * Create a tokenizer that tokenizes the recognized character range
16    * to a TypedToken object.
17    * @param type the token type.
18    * @return the Tokenizer object.
19    */
20   public static <T> Tokenizer forTypedToken(final T type){
21     return new Tokenizer(){
22       public Object JavaDoc toToken(CharSequence JavaDoc cs, int from, int len) {
23         return Tokens.createTypedToken(cs.subSequence(from, from+len).toString(), type);
24       }
25     };
26   }
27   private static final Tokenizer reserved = forTypedToken(TokenType.Reserved);
28   private static final Tokenizer word = forTypedToken(TokenType.Word);
29   private static final Tokenizer integer = forTypedToken(TokenType.Integer);
30   private static final Tokenizer decimal = forTypedToken(TokenType.Decimal);
31   /**
32    * Get the Tokenizer object that creates a reserved word token.
33    */
34   public static Tokenizer forReservedWord(){
35     return reserved;
36   }
37   /**
38    * Get the Tokenizer object that creates a word token.
39    */
40   public static Tokenizer forWord(){
41     return word;
42   }
43   /**
44    * Get the Tokenizer object that creates an integer literal token.
45    */
46   public static Tokenizer forInteger(){
47     return integer;
48   }
49   /**
50    * Get the Tokenizer object that creates a decimal number literal token.
51    */
52   public static Tokenizer forDecimal(){
53     return decimal;
54   }
55   /**
56    * Creates a tokenizer that's gonna tokenize a single quoted character literal possibly with escape character '\'
57    * @return the tokenizer instance.
58    */
59   public static Tokenizer forChar(){
60     return TokenCharLiteral.getTokenizer();
61   }
62   /**
63    * Get the Tokenizer object that creates a string object.
64    */
65   public static Tokenizer forString(){
66     return TokenString.getTokenizer();
67   }
68   /**
69    * Get the Tokenizer object that interprets the recognized character range
70    * as a decimal integer and translate it to a long value.
71    */
72   public static Tokenizer forDecLong(){
73     return TokenLong.getDecTokenizer();
74   }
75   /**
76    * Get the Tokenizer object that interprets the recognized character range
77    * as a hex integer and translate it to a long value.
78    */
79   public static Tokenizer forHexLong(){
80     return TokenLong.getHexTokenizer();
81   }
82   /**
83    * Get the Tokenizer object that interprets the recognized character range
84    * as a oct integer and translate it to a long value.
85    */
86   public static Tokenizer forOctLong(){
87     return TokenLong.getOctTokenizer();
88   }
89   /**
90    * Get the Tokenizer object that converts a string literal quoted by '"'
91    * to a string object.
92    * back-slash character is escaped.
93    */
94   public static Tokenizer forSimpleStringLiteral(){
95     return TokenStringLiteral.getDoubleQuoteTokenizer();
96   }
97   /**
98    * Get the Tokenizer object that converts a sql string literal quoted by single quote
99    * to a string object.
100    * double single quote is interpreted as one single quote.
101    */
102   public static Tokenizer forSqlStringLiteral(){
103     return TokenStringLiteral.getSqlTokenizer();
104   }
105   /**
106    * Get the Tokenizer object that converts a string literal quoted by a pair of
107    * opening and closing characters.
108    * The Tokenizer result is a {@link TokenQuoted} object.
109    */
110   public static Tokenizer forQuotedString(char open, char close){
111     return TokenQuoted.getTokenizer(open, close);
112   }
113   /**
114    * Get the Tokenizer object that converts a string literal quoted by a pair of
115    * opening and closing strings.
116    * The Tokenizer result is a {@link TokenQuoted} object.
117    */
118   public static Tokenizer forQuotedString(String JavaDoc open, String JavaDoc close){
119     return TokenQuoted.getTokenizer(open, close);
120   }
121 }
122
Popular Tags