Token


1   /*
2    * Token.java
3    *
4    * This library is free software; you can redistribute it and/or
5    * modify it under the terms of the GNU Lesser General Public License
6    * as published by the Free Software Foundation; either version 2.1
7    * of the License, or (at your option) any later version.
8    *
9    * This library is distributed in the hope that it will be useful,
10   * but WITHOUT ANY WARRANTY; without even the implied warranty of
11   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12   * Lesser General Public License for more details.
13   *
14   * You should have received a copy of the GNU Lesser General Public
15   * License along with this library; if not, write to the Free
16   * Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
17   * MA 02111-1307, USA.
18   *
19   * Copyright (c) 2003-2005 Per Cederberg. All rights reserved.
20   */
21  
22  package net.percederberg.grammatica.parser;
23  
24  /**
25   * A token node. This class represents a token (i.e. a set of adjacent
26   * characters) in a parse tree. The tokens are created by a tokenizer,
27   * that groups characters together into tokens according to a set of
28   * token patterns.
29   *
30   * @author   Per Cederberg, <per at percederberg dot net>
31   * @version  1.4
32   */
33  public class Token extends Node {
34  
35      /**
36       * The token pattern used for this token.
37       */
38      private TokenPattern pattern;
39  
40      /**
41       * The characters that constitute this token. This is normally
42       * referred to as the token image.
43       */
44      private String   image;
45  
46      /**
47       * The line number of the first character in the token image.
48       */
49      private int startLine;
50  
51      /**
52       * The column number of the first character in the token image.
53       */
54      private int startColumn;
55  
56      /**
57       * The line number of the last character in the token image.
58       */
59      private int endLine;
60  
61      /**
62       * The column number of the last character in the token image.
63       */
64      private int endColumn;
65  
66      /**
67       * The previous token in the list of tokens.
68       */
69      private Token previous = null;
70  
71      /**
72       * The next token in the list of tokens.
73       */
74      private Token next = null;
75  
76      /**
77       * Creates a new token.
78       *
79       * @param pattern        the token pattern
80       * @param image          the token image (i.e. characters)
81       * @param line           the line number of the first character
82       * @param col            the column number of the first character
83       */
84      public Token(TokenPattern pattern, String   image, int line, int col) {
85          this.pattern = pattern;
86          this.image = image;
87          this.startLine = line;
88          this.startColumn = col;
89          this.endLine = line;
90          this.endColumn = col + image.length() - 1;
91          for (int pos = 0; image.indexOf('\n', pos) >= 0;) {
92              pos = image.indexOf('\n', pos) + 1;
93              this.endLine++;
94              this.endColumn = image.length() - pos;
95          }
96      }
97  
98      /**
99       * Returns the token (pattern) id. This value is set as a unique
100      * identifier when creating the token pattern to simplify later
101      * identification.
102      *
103      * @return the token id
104      */
105     public int getId() {
106         return pattern.getId();
107     }
108 
109     /**
110      * Returns the token node name.
111      *
112      * @return the token node name
113      */
114     public String   getName() {
115         return pattern.getName();
116     }
117 
118     /**
119      * Returns the token image. The token image consists of the
120      * input characters matched to form this token.
121      *
122      * @return the token image
123      */
124     public String   getImage() {
125         return image;
126     }
127 
128     /**
129      * The line number of the first character in the token image.
130      *
131      * @return the line number of the first token character
132      */
133     public int getStartLine() {
134         return startLine;
135     }
136 
137     /**
138      * The column number of the first character in the token image.
139      *
140      * @return the column number of the first token character
141      */
142     public int getStartColumn() {
143         return startColumn;
144     }
145 
146     /**
147      * The line number of the last character in the token image.
148      *
149      * @return the line number of the last token character
150      */
151     public int getEndLine() {
152         return endLine;
153     }
154 
155     /**
156      * The column number of the last character in the token image.
157      *
158      * @return the column number of the last token character
159      */
160     public int getEndColumn() {
161         return endColumn;
162     }
163 
164     /**
165      * Returns the token pattern.
166      *
167      * @return the token pattern
168      */
169     TokenPattern getPattern() {
170         return pattern;
171     }
172 
173     /**
174      * Returns the previuos token. The previous token may be a token
175      * that has been ignored in the parsing. Note that if the token
176      * list feature hasn't been used in the tokenizer, this method
177      * will always return null. By default the token list feature is
178      * not used.
179      *
180      * @return the previous token, or
181      *         null if no such token is available
182      *
183      * @see #getNextToken
184      * @see Tokenizer#getUseTokenList
185      * @see Tokenizer#setUseTokenList
186      *
187      * @since 1.4
188      */
189     public Token getPreviousToken() {
190         return previous;
191     }
192 
193     /**
194      * Sets the previous token in the token list. This method will
195      * also modify the token specified to have this token as it's
196      * next token.
197      *
198      * @param previous       the previous token, or null for none
199      *
200      * @since 1.4
201      */
202     void setPreviousToken(Token previous) {
203         if (this.previous != null) {
204             this.previous.next = null;
205         }
206         this.previous = previous;
207         if (previous != null) {
208             previous.next = this;
209         }
210     }
211 
212     /**
213      * Returns the next token. The next token may be a token that has
214      * been ignored in the parsing. Note that if the token list
215      * feature hasn't been used in the tokenizer, this method will
216      * always return null. By default the token list feature is not
217      * used.
218      *
219      * @return the next token, or
220      *         null if no such token is available
221      *
222      * @see #getPreviousToken
223      * @see Tokenizer#getUseTokenList
224      * @see Tokenizer#setUseTokenList
225      *
226      * @since 1.4
227      */
228     public Token getNextToken() {
229         return next;
230     }
231 
232     /**
233      * Sets the next token in the token list. This method will also
234      * modify the token specified to have this token as it's
235      * previous token.
236      *
237      * @param next           the next token, or null for none
238      *
239      * @since 1.4
240      */
241     void setNextToken(Token next) {
242         if (this.next != null) {
243             this.next.previous = null;
244         }
245         this.next = next;
246         if (next != null) {
247             next.previous = this;
248         }
249     }
250 
251     /**
252      * Returns a detailed string representation of this token.
253      *
254      * @return a detailed string representation of this token
255      *
256      * @see java.lang.Object#toString()
257      */
258     public String   toString() {
259         StringBuffer    buffer = new StringBuffer  ();
260         int           newline = image.indexOf('\n');
261 
262         buffer.append(pattern.getName());
263         buffer.append("(");
264         buffer.append(pattern.getId());
265         buffer.append("): \"");
266         if (newline >= 0) {
267             if (newline > 0 && image.charAt(newline - 1) == '\r') {
268                 newline--;
269             }
270             buffer.append(image.substring(0, newline));
271             buffer.append("(...)");
272         } else {
273             buffer.append(image);
274         }
275         buffer.append("\", line: ");
276         buffer.append(startLine);
277         buffer.append(", col: ");
278         buffer.append(startColumn);
279 
280         return buffer.toString();
281     }
282 
283     /**
284      * Returns a short string representation of this token. The string
285      * will only contain the token image and possibly the token
286      * pattern name.
287      *
288      * @return a short string representation of this token
289      */
290     public String   toShortString() {
291         StringBuffer    buffer = new StringBuffer  ();
292         int           newline = image.indexOf('\n');
293 
294         buffer.append('"');
295         if (newline >= 0) {
296             if (newline > 0 && image.charAt(newline - 1) == '\r') {
297                 newline--;
298             }
299             buffer.append(image.substring(0, newline));
300             buffer.append("(...)");
301         } else {
302             buffer.append(image);
303         }
304         buffer.append('"');
305         if (pattern.getType() == TokenPattern.REGEXP_TYPE) {
306             buffer.append(" <");
307             buffer.append(pattern.getName());
308             buffer.append(">");
309         }
310 
311         return buffer.toString();
312     }
313 }
314
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags