GrammarTokenizer


1   /*
2    * GrammarTokenizer.java
3    *
4    * THIS FILE HAS BEEN GENERATED AUTOMATICALLY. DO NOT EDIT!
5    *
6    * This library is free software; you can redistribute it and/or
7    * modify it under the terms of the GNU Lesser General Public License
8    * as published by the Free Software Foundation; either version 2.1
9    * of the License, or (at your option) any later version.
10   *
11   * This library is distributed in the hope that it will be useful,
12   * but WITHOUT ANY WARRANTY; without even the implied warranty of
13   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14   * Lesser General Public License for more details.
15   *
16   * You should have received a copy of the GNU Lesser General Public
17   * License along with this library; if not, write to the Free
18   * Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
19   * MA 02111-1307, USA.
20   *
21   * Copyright (c) 2003-2005 Per Cederberg. All rights reserved.
22   */
23  
24  package net.percederberg.grammatica;
25  
26  import java.io.Reader  ;
27  
28  import net.percederberg.grammatica.parser.ParserCreationException;
29  import net.percederberg.grammatica.parser.TokenPattern;
30  import net.percederberg.grammatica.parser.Tokenizer;
31  
32  /**
33   * A character stream tokenizer.
34   *
35   * @author   Per Cederberg, <per at percederberg dot net>
36   * @version  1.5
37   */
38  class GrammarTokenizer extends Tokenizer {
39  
40      /**
41       * Creates a new tokenizer for the specified input stream.
42       *
43       * @param input          the input stream to read
44       *
45       * @throws ParserCreationException if the tokenizer couldn't be
46       *             initialized correctly
47       */
48      public GrammarTokenizer(Reader   input)
49          throws ParserCreationException {
50  
51          super(input, false);
52          createPatterns();
53      }
54  
55      /**
56       * Initializes the tokenizer by creating all the token patterns.
57       *
58       * @throws ParserCreationException if the tokenizer couldn't be
59       *             initialized correctly
60       */
61      private void createPatterns() throws ParserCreationException {
62          TokenPattern  pattern;
63  
64          pattern = new TokenPattern(GrammarConstants.HEADER,
65                                     "HEADER",
66                                     TokenPattern.STRING_TYPE,
67                                     "%header%");
68          addPattern(pattern);
69  
70          pattern = new TokenPattern(GrammarConstants.TOKENS,
71                                     "TOKENS",
72                                     TokenPattern.STRING_TYPE,
73                                     "%tokens%");
74          addPattern(pattern);
75  
76          pattern = new TokenPattern(GrammarConstants.PRODUCTIONS,
77                                     "PRODUCTIONS",
78                                     TokenPattern.STRING_TYPE,
79                                     "%productions%");
80          addPattern(pattern);
81  
82          pattern = new TokenPattern(GrammarConstants.IGNORE,
83                                     "IGNORE",
84                                     TokenPattern.REGEXP_TYPE,
85                                     "%ignore([^%]*)%");
86          addPattern(pattern);
87  
88          pattern = new TokenPattern(GrammarConstants.ERROR,
89                                     "ERROR",
90                                     TokenPattern.REGEXP_TYPE,
91                                     "%error([^%]*)%");
92          addPattern(pattern);
93  
94          pattern = new TokenPattern(GrammarConstants.UNTERMINATED_DIRECTIVE,
95                                     "UNTERMINATED_DIRECTIVE",
96                                     TokenPattern.REGEXP_TYPE,
97                                     "%[^%\\n\\r]*");
98          pattern.setError("unterminated directive");
99          addPattern(pattern);
100 
101         pattern = new TokenPattern(GrammarConstants.EQUALS,
102                                    "EQUALS",
103                                    TokenPattern.STRING_TYPE,
104                                    "=");
105         addPattern(pattern);
106 
107         pattern = new TokenPattern(GrammarConstants.LEFT_PAREN,
108                                    "LEFT_PAREN",
109                                    TokenPattern.STRING_TYPE,
110                                    "(");
111         addPattern(pattern);
112 
113         pattern = new TokenPattern(GrammarConstants.RIGHT_PAREN,
114                                    "RIGHT_PAREN",
115                                    TokenPattern.STRING_TYPE,
116                                    ")");
117         addPattern(pattern);
118 
119         pattern = new TokenPattern(GrammarConstants.LEFT_BRACE,
120                                    "LEFT_BRACE",
121                                    TokenPattern.STRING_TYPE,
122                                    "{");
123         addPattern(pattern);
124 
125         pattern = new TokenPattern(GrammarConstants.RIGHT_BRACE,
126                                    "RIGHT_BRACE",
127                                    TokenPattern.STRING_TYPE,
128                                    "}");
129         addPattern(pattern);
130 
131         pattern = new TokenPattern(GrammarConstants.LEFT_BRACKET,
132                                    "LEFT_BRACKET",
133                                    TokenPattern.STRING_TYPE,
134                                    "[");
135         addPattern(pattern);
136 
137         pattern = new TokenPattern(GrammarConstants.RIGHT_BRACKET,
138                                    "RIGHT_BRACKET",
139                                    TokenPattern.STRING_TYPE,
140                                    "]");
141         addPattern(pattern);
142 
143         pattern = new TokenPattern(GrammarConstants.QUESTION_MARK,
144                                    "QUESTION_MARK",
145                                    TokenPattern.STRING_TYPE,
146                                    "?");
147         addPattern(pattern);
148 
149         pattern = new TokenPattern(GrammarConstants.PLUS_SIGN,
150                                    "PLUS_SIGN",
151                                    TokenPattern.STRING_TYPE,
152                                    "+");
153         addPattern(pattern);
154 
155         pattern = new TokenPattern(GrammarConstants.ASTERISK,
156                                    "ASTERISK",
157                                    TokenPattern.STRING_TYPE,
158                                    "*");
159         addPattern(pattern);
160 
161         pattern = new TokenPattern(GrammarConstants.VERTICAL_BAR,
162                                    "VERTICAL_BAR",
163                                    TokenPattern.STRING_TYPE,
164                                    "|");
165         addPattern(pattern);
166 
167         pattern = new TokenPattern(GrammarConstants.SEMICOLON,
168                                    "SEMICOLON",
169                                    TokenPattern.STRING_TYPE,
170                                    ";");
171         addPattern(pattern);
172 
173         pattern = new TokenPattern(GrammarConstants.IDENTIFIER,
174                                    "IDENTIFIER",
175                                    TokenPattern.REGEXP_TYPE,
176                                    "[A-Za-z][A-Za-z0-9_]*");
177         addPattern(pattern);
178 
179         pattern = new TokenPattern(GrammarConstants.QUOTED_STRING,
180                                    "QUOTED_STRING",
181                                    TokenPattern.REGEXP_TYPE,
182                                    "(\"[^\"]*\")|('[^']*')");
183         addPattern(pattern);
184 
185         pattern = new TokenPattern(GrammarConstants.REGEXP,
186                                    "REGEXP",
187                                    TokenPattern.REGEXP_TYPE,
188                                    "<<([^\\\\>]|(\\\\.)|(>[^>]))*>>");
189         addPattern(pattern);
190 
191         pattern = new TokenPattern(GrammarConstants.SINGLE_LINE_COMMENT,
192                                    "SINGLE_LINE_COMMENT",
193                                    TokenPattern.REGEXP_TYPE,
194                                    "//.*");
195         pattern.setIgnore();
196         addPattern(pattern);
197 
198         pattern = new TokenPattern(GrammarConstants.MULTI_LINE_COMMENT,
199                                    "MULTI_LINE_COMMENT",
200                                    TokenPattern.REGEXP_TYPE,
201                                    "/\\*(.|\\n)*?\\*/");
202         pattern.setIgnore();
203         addPattern(pattern);
204 
205         pattern = new TokenPattern(GrammarConstants.WHITESPACE,
206                                    "WHITESPACE",
207                                    TokenPattern.REGEXP_TYPE,
208                                    "[ \\t\\n\\r]+");
209         pattern.setIgnore();
210         addPattern(pattern);
211     }
212 }
213
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags