Token


1   package net.sf.saxon.expr;
2   
3   import java.util.HashMap  ;
4   
5   /**
6    * This class holds static constants and methods defining the lexical tokens used in
7    * XPath and XQuery, and associated keywords.
8    */
9   
10  public abstract class Token {
11  
12      /**
13       * Token numbers. Those in the range 0 to 100 are tokens that can be followed
14       * by a name or expression; those in the range 101 to 200 are tokens that can be
15       * followed by an binary operator.
16       */
17  
18      /**
19       * Pseudo-token representing the end of the expression
20       */
21      public static final int EOF = 0;
22      /**
23       * "union" or "|" token
24       */
25      public static final int UNION = 1;
26      /**
27       * Forwards "/"
28       */
29      public static final int SLASH = 2;
30      /**
31       * At token, "@"
32       */
33      public static final int AT = 3;
34      /**
35       * Left square bracket
36       */
37      public static final int LSQB = 4;
38      /**
39       * Left parenthesis
40       */
41      public static final int LPAR = 5;
42      /**
43       * Equals token ("=")
44       */
45      public static final int EQUALS = 6;
46      /**
47       * Comma token
48       */
49      public static final int COMMA = 7;
50      /**
51       * Double forwards slash, "//"
52       */
53      public static final int SLSL = 8;
54      /**
55       * Operator "or"
56       */
57      public static final int OR = 9;
58      /**
59       * Operator "and"
60       */
61      public static final int AND = 10;
62      /**
63       * Operator ">"
64       */
65      public static final int GT = 11;
66      /**
67       * Operator "<"
68       */
69      public static final int LT = 12;
70      /**
71       * Operator ">="
72       */
73      public static final int GE = 13;
74      /**
75       * Operator "<="
76       */
77      public static final int LE = 14;
78      /**
79       * Operator "+"
80       */
81      public static final int PLUS = 15;
82      /**
83       * Binary minus operator
84       */
85      public static final int MINUS = 16;
86      /**
87       * Multiply operator, "*" when used in an operator context
88       */
89      public static final int MULT = 17;
90      /**
91       * Operator "div"
92       */
93      public static final int DIV = 18;
94      /**
95       * Operator "mod"
96       */
97      public static final int MOD = 19;
98      /**
99       * Operator "is"
100      */
101     public static final int IS = 20;
102     /**
103      * "$" symbol
104      */
105     public static final int DOLLAR = 21;
106     /**
107      * Operator not-equals. That is, "!="
108      */
109     public static final int NE = 22;
110     /**
111      * Operator "intersect"
112      */
113     public static final int INTERSECT = 23;
114     /**
115      * Operator "except"
116      */
117     public static final int EXCEPT = 24;
118     /**
119      * Keyword "return"
120      */
121     public static final int RETURN = 25;
122     /**
123      * Ketword "then"
124      */
125     public static final int THEN = 26;
126     /**
127      * Keyword "else"
128      */
129     public static final int ELSE = 27;
130     /**
131      * Keyword "where"
132      */
133     public static final int WHERE = 28;
134     /**
135      * Operator "to"
136      */
137     public static final int TO = 29;
138     /**
139      * Keyword "in"
140      */
141     public static final int IN = 30;
142     /**
143      * Keyword "some"
144      */
145     public static final int SOME = 31;
146     /**
147      * Keyword "every"
148      */
149     public static final int EVERY = 32;
150     /**
151      * Keyword "satisfies"
152      */
153     public static final int SATISFIES = 33;
154     /**
155      * Token representing the name of a function and the following "(" symbol
156      */
157     public static final int FUNCTION = 34;
158     /**
159      * Token representing the name of an axis and the following "::" symbol
160      */
161     public static final int AXIS = 35;
162     /**
163      * Keyword "if"
164      */
165     public static final int IF = 36;
166     /**
167      * Operator "<<"
168      */
169     public static final int PRECEDES = 37;
170     /**
171      * Operator ">>"
172      */
173     public static final int FOLLOWS = 38;
174     /**
175      * "::" symbol
176      */
177     public static final int COLONCOLON = 39;
178     /**
179      * ":*" symbol
180      */
181     public static final int COLONSTAR = 40;
182     /**
183      * operator "instance of"
184      */
185     public static final int INSTANCE_OF = 41;
186     /**
187      * operator "cast as"
188      */
189     public static final int CAST_AS = 42;
190     /**
191      * operator "treat as"
192      */
193     public static final int TREAT_AS = 43;
194     /**
195      * operator "eq"
196      */
197     public static final int FEQ = 44;       // "Fortran" style comparison operators eq, ne, etc
198     /**
199      * operator "ne"
200      */
201     public static final int FNE = 45;
202     /**
203      * operator "gt"
204      */
205     public static final int FGT = 46;
206     /**
207      * operator "lt"
208      */
209     public static final int FLT = 47;
210     /**
211      * operator "ge"
212      */
213     public static final int FGE = 48;
214     /**
215      * opeartor "le"
216      */
217     public static final int FLE = 49;
218     /**
219      * operator "idiv"
220      */
221     public static final int IDIV = 50;
222     /**
223      * operator "castable as"
224      */
225     public static final int CASTABLE_AS = 51;
226     /**
227       * ":=" symbol (XQuery only)
228       */
229     public static final int ASSIGN = 52;
230     /**
231      * "{" symbol (XQuery only)
232      */
233     public static final int LCURLY = 53;
234     /**
235      * composite token: <keyword "{"> (XQuery only)
236      */
237     public static final int KEYWORD_CURLY = 54;
238     /**
239      * composite token <'element' QNAME> (XQuery only)
240      */
241     public static final int ELEMENT_QNAME = 55;
242     /**
243      * composite token <'attribute' QNAME> (XQuery only)
244      */
245     public static final int ATTRIBUTE_QNAME = 56;
246     /**
247      * composite token <'pi' QNAME> (XQuery only)
248      */
249     public static final int PI_QNAME = 57;
250     /**
251      * Keyword "typeswitch"
252      */
253     public static final int TYPESWITCH = 58;
254     /**
255      * Keyword "case"
256      */
257     public static final int CASE = 59;
258     /**
259      * Keyword "default"
260      */
261     public static final int DEFAULT = 60;
262      /**
263      * Node kind, e.g. "node()" or "comment()"
264      */
265     public static final int NODEKIND = 61;
266 
267 
268     // The following tokens are used only in the query prolog. They are categorized
269     // as operators on the basis that a following name is treated as a name rather than
270     // an operator.
271 
272 
273     /**
274      * "xquery version"
275      */
276     public static final int XQUERY_VERSION = 70;
277     /**
278      * "declare namespace"
279      */
280     public static final int DECLARE_NAMESPACE = 71;
281     /**
282      * "declare default"
283      */
284     public static final int DECLARE_DEFAULT = 72;
285     /**
286      * "declare construction"
287      */
288     public static final int DECLARE_CONSTRUCTION = 73;
289     /**
290      * "declare base-uri"
291      */
292     public static final int DECLARE_BASEURI = 74;
293     /**
294      * "declare boundary-space"
295      */
296     public static final int DECLARE_BOUNDARY_SPACE = 75;
297     /**
298      * "import schema"
299      */
300     public static final int IMPORT_SCHEMA = 76;
301     /**
302      * "import module"
303      */
304     public static final int IMPORT_MODULE = 77;
305     /**
306      * "define variable"
307      */
308     public static final int DECLARE_VARIABLE = 78;
309     /**
310      * "define function"
311      */
312     public static final int DECLARE_FUNCTION = 79;
313     /**
314      * "module namespace"
315      */
316     public static final int MODULE_NAMESPACE = 80;
317     /**
318      * Various compound symbols supporting XQuery validation expression
319      */
320     public static final int VALIDATE = 81;
321     public static final int VALIDATE_STRICT = 82;
322     public static final int VALIDATE_LAX = 83;
323 
324     /**
325      * "declare xmlspace"
326      */
327     public static final int DECLARE_ORDERING = 84;
328 
329     /**
330      * "declare copy-namespaces"
331      */
332     public static final int DECLARE_COPY_NAMESPACES = 85;
333     /**
334      * "declare option"
335      */
336     public static final int DECLARE_OPTION = 86;
337     /**
338      * semicolon separator
339      */
340     public static final int SEMICOLON = 90;
341 
342 
343     /**
344      * Constant identifying the token number of the last token to be classified as an operator
345      */
346     static int LAST_OPERATOR = 100;
347 
348     // Tokens that set "operator" context, so an immediately following "div" is recognized
349     // as an operator, not as an element name
350 
351     /**
352      * Name token (a QName, in general)
353      */
354     public static final int NAME = 101;
355     /**
356      * String literal
357      */
358     public static final int STRING_LITERAL = 102;
359     /**
360      * Right square bracket
361      */
362     public static final int RSQB = 103;
363     /**
364      * Right parenthesis
365      */
366     public static final int RPAR = 104;
367     /**
368      * "." symbol
369      */
370     public static final int DOT = 105;
371     /**
372      * ".." symbol
373      */
374     public static final int DOTDOT = 106;
375     /**
376      * "*" symbol when used as a wildcard
377      */
378     public static final int STAR = 107;
379     /**
380      * "prefix:*" token
381      */
382     public static final int PREFIX = 108;    // e.g. prefix:*
383     /**
384      * Numeric literal
385      */
386     public static final int NUMBER = 109;
387 
388     /**
389      * "for" keyword
390      */
391     public static final int FOR = 111;
392     /**
393      * "*:local-name" token
394      */
395     public static final int SUFFIX = 112;    // e.g. *:suffix
396     /**
397      * Question mark symbol. That is, "?"
398      */
399     public static final int QMARK = 113;
400     /**
401      * "}" symbol (XQuery only)
402      */
403     public static final int RCURLY = 115;
404     /**
405      * "let" keyword (XQuery only)
406      */
407     public static final int LET = 116;
408     /**
409      * "<" at the start of a tag (XQuery only). The pseudo-XML syntax that
410      * follows is read character-by-character by the XQuery parser
411      */
412     public static final int TAG = 117;
413     /**
414      * A token representing an XQuery pragma.
415      * This construct "(# .... #)" is regarded as a single token, for the QueryParser to sort out.
416      */
417     public static final int PRAGMA = 118;
418 
419 
420     /**
421      * Unary minus sign
422      */
423     public static final int NEGATE = 199;    // unary minus: not actually a token, but we
424                                              // use token numbers to identify operators.
425 
426 
427     /**
428      * The following strings are used to represent tokens in error messages
429      */
430 
431     public static String  [] tokens = new String  [200];
432     static {
433         tokens [ EOF ] = "<eof>";
434         tokens [ UNION ] = "|";
435         tokens [ SLASH ] = "/";
436         tokens [ AT ] = "@";
437         tokens [ LSQB ] = "[";
438         tokens [ LPAR ] = "(";
439         tokens [ EQUALS ] = "=";
440         tokens [ COMMA ] = ",";
441         tokens [ SLSL ] = "//";
442         tokens [ OR ] = "or";
443         tokens [ AND ] = "and";
444         tokens [ GT ] = ">";
445         tokens [ LT ] = "<";
446         tokens [ GE ] = ">=";
447         tokens [ LE ] = "<=";
448         tokens [ PLUS ] = "+";
449         tokens [ MINUS ] = "-";
450         tokens [ MULT ] = "*";
451         tokens [ DIV ] = "div";
452         tokens [ MOD ] = "mod";
453         tokens [ IS ] = "is";
454         tokens [ DOLLAR ] = "$";
455         tokens [ NE ] = "!=";
456         tokens [ INTERSECT ] = "intersect";
457         tokens [ EXCEPT ] = "except";
458         tokens [ RETURN ] = "return";
459         tokens [ THEN ] = "then";
460         tokens [ ELSE ] = "else";
461         //tokens [ ISNOT ] = "isnot";
462         tokens [ TO ] = "to";
463         tokens [ IN ] = "in";
464         tokens [ SOME ] = "some";
465         tokens [ EVERY ] = "every";
466         tokens [ SATISFIES ] = "satisfies";
467         tokens [ FUNCTION ] = "<function>(";
468         tokens [ AXIS ] = "<axis>";
469         tokens [ IF ] = "if(";
470         tokens [ PRECEDES ] = "<<";
471         tokens [ FOLLOWS ] = ">>";
472         tokens [ COLONCOLON ] = "::";
473         tokens [ COLONSTAR ] = ":*";
474         tokens [ INSTANCE_OF ] = "instance of";
475         tokens [ CAST_AS ] = "cast as";
476         tokens [ TREAT_AS ] = "treat as";
477         tokens [ FEQ ] = "eq";
478         tokens [ FNE ] = "ne";
479         tokens [ FGT ] = "gt";
480         tokens [ FGE ] = "ge";
481         tokens [ FLT ] = "lt";
482         tokens [ FLE ] = "le";
483         tokens [ IDIV ] = "idiv";
484         tokens [ CASTABLE_AS ] = "castable as";
485         tokens [ ASSIGN ] = ":=";
486         tokens [ TYPESWITCH ] = "typeswitch";
487         tokens [ CASE ] = "case";
488         tokens [ DEFAULT ] = "default";
489 
490 
491         tokens [ NAME ] = "<name>";
492         tokens [ STRING_LITERAL ] = "<string-literal>";
493         tokens [ RSQB ] = "]";
494         tokens [ RPAR ] = ")";
495         tokens [ DOT ] = ".";
496         tokens [ DOTDOT ] = "..";
497         tokens [ STAR ] = "*";
498         tokens [ PREFIX ] = "<prefix:*>";
499         tokens [ NUMBER ] = "<numeric-literal>";
500         tokens [ NODEKIND ] = "<node-type>()";
501         tokens [ FOR ] = "for";
502         tokens [ SUFFIX ] = "<*:local-name>";
503         tokens [ QMARK ] = "?";
504         tokens [ LCURLY ] = "{";
505         tokens [ KEYWORD_CURLY ] = "<keyword> {";
506         tokens [ RCURLY ] = "}";
507         tokens [ LET ] = "let";
508         tokens [ VALIDATE ] = "validate {";
509         tokens [ TAG ] = "<element>";
510         tokens [ PRAGMA ] = "(# ... #)";
511         tokens [ SEMICOLON ] = ";";
512         tokens [ NEGATE ] = "-";
513     }
514 
515     /**
516      * Lookup table for composite (two-keyword) tokens
517      */
518     public static HashMap   doubleKeywords = new HashMap  (30);
519     /**
520      * Pseudo-token representing the start of the expression
521      */
522     public static final int UNKNOWN = -1;
523 
524     private Token() {
525     }
526 
527     static {
528         mapDouble("instance of", INSTANCE_OF);
529         mapDouble("cast as", CAST_AS);
530         mapDouble("treat as", TREAT_AS);
531         mapDouble("castable as", CASTABLE_AS);
532         mapDouble("xquery version", XQUERY_VERSION);
533         mapDouble("declare namespace", DECLARE_NAMESPACE);
534         mapDouble("declare default", DECLARE_DEFAULT);
535         mapDouble("declare construction", DECLARE_CONSTRUCTION);
536         mapDouble("declare base-uri", DECLARE_BASEURI);
537         mapDouble("declare boundary-space", DECLARE_BOUNDARY_SPACE);
538         mapDouble("declare ordering", DECLARE_ORDERING);
539         mapDouble("declare copy-namespaces", DECLARE_COPY_NAMESPACES);
540         mapDouble("declare option", DECLARE_OPTION);
541         mapDouble("import schema", IMPORT_SCHEMA);
542         mapDouble("import module", IMPORT_MODULE);
543         mapDouble("declare variable", DECLARE_VARIABLE);
544         mapDouble("declare function", DECLARE_FUNCTION);
545         mapDouble("module namespace", MODULE_NAMESPACE);
546         mapDouble("validate strict", VALIDATE_STRICT);
547         mapDouble("validate lax", VALIDATE_LAX);
548 
549     }
550 
551     private static void mapDouble(String   doubleKeyword, int token) {
552         doubleKeywords.put(doubleKeyword, new Integer  (token));
553         tokens[token] = doubleKeyword;
554     }
555 
556     /**
557     * Return the inverse of a relational operator, so that "a op b" can be
558     * rewritten as "b inverse(op) a"
559     */
560 
561     public static final int inverse(int operator) {
562         switch(operator) {
563             case LT:
564                 return GT;
565             case LE:
566                 return GE;
567             case GT:
568                 return LT;
569             case GE:
570                 return LE;
571             case FLT:
572                 return FGT;
573             case FLE:
574                 return FGE;
575             case FGT:
576                 return FLT;
577             case FGE:
578                 return FLE;
579             default:
580                 return operator;
581         }
582     }
583 }
584 
585 //
586 // The contents of this file are subject to the Mozilla Public License Version 1.0 (the "License");
587 // you may not use this file except in compliance with the License. You may obtain a copy of the
588 // License at http://www.mozilla.org/MPL/
589 //
590 // Software distributed under the License is distributed on an "AS IS" basis,
591 // WITHOUT WARRANTY OF ANY KIND, either express or implied.
592 // See the License for the specific language governing rights and limitations under the License.
593 //
594 // The Original Code is: all this file.
595 //
596 // The Initial Developer of the Original Code is Michael H. Kay.
597 //
598 // Portions created by (your name) are Copyright (C) (your legal entity). All Rights Reserved.
599 //
600 // Contributor(s): none.
601 //
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags