KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > antlr > TokenBuffer


1 package antlr;
2
3 /* ANTLR Translator Generator
4  * Project led by Terence Parr at http://www.jGuru.com
5  * Software rights: http://www.antlr.org/RIGHTS.html
6  *
7  * $Id: //depot/code/org.antlr/main/main/antlr/TokenBuffer.java#6 $
8  */
9
10 /**A Stream of Token objects fed to the parser from a Tokenizer that can
11  * be rewound via mark()/rewind() methods.
12  * <p>
13  * A dynamic array is used to buffer up all the input tokens. Normally,
14  * "k" tokens are stored in the buffer. More tokens may be stored during
15  * guess mode (testing syntactic predicate), or when LT(i>k) is referenced.
16  * Consumption of tokens is deferred. In other words, reading the next
17  * token is not done by conume(), but deferred until needed by LA or LT.
18  * <p>
19  *
20  * @see antlr.Token
21  * @see antlr.Tokenizer
22  * @see antlr.TokenQueue
23  */
24
25 import java.io.IOException JavaDoc;
26
27 public class TokenBuffer {
28
29     // Token source
30 protected TokenStream input;
31
32     // Number of active markers
33 int nMarkers = 0;
34
35     // Additional offset used when markers are active
36 int markerOffset = 0;
37
38     // Number of calls to consume() since last LA() or LT() call
39 int numToConsume = 0;
40
41     // Circular queue
42 TokenQueue queue;
43
44     /** Create a token buffer */
45     public TokenBuffer(TokenStream input_) {
46         input = input_;
47         queue = new TokenQueue(1);
48     }
49
50     /** Reset the input buffer to empty state */
51     public final void reset() {
52         nMarkers = 0;
53         markerOffset = 0;
54         numToConsume = 0;
55         queue.reset();
56     }
57
58     /** Mark another token for deferred consumption */
59     public final void consume() {
60         numToConsume++;
61     }
62
63     /** Ensure that the token buffer is sufficiently full */
64     private final void fill(int amount) throws TokenStreamException {
65         syncConsume();
66         // Fill the buffer sufficiently to hold needed tokens
67 while (queue.nbrEntries < amount + markerOffset) {
68             // Append the next token
69 queue.append(input.nextToken());
70         }
71     }
72
73     /** return the Tokenizer (needed by ParseView) */
74     public TokenStream getInput() {
75         return input;
76     }
77
78     /** Get a lookahead token value */
79     public final int LA(int i) throws TokenStreamException {
80         fill(i);
81         return queue.elementAt(markerOffset + i - 1).type;
82     }
83
84     /** Get a lookahead token */
85     public final Token LT(int i) throws TokenStreamException {
86         fill(i);
87         return queue.elementAt(markerOffset + i - 1);
88     }
89
90     /**Return an integer marker that can be used to rewind the buffer to
91      * its current state.
92      */
93     public final int mark() {
94         syncConsume();
95 //System.out.println("Marking at " + markerOffset);
96 //try { for (int i = 1; i <= 2; i++) { System.out.println("LA("+i+")=="+LT(i).getText()); } } catch (ScannerException e) {}
97 nMarkers++;
98         return markerOffset;
99     }
100
101     /**Rewind the token buffer to a marker.
102      * @param mark Marker returned previously from mark()
103      */
104     public final void rewind(int mark) {
105         syncConsume();
106         markerOffset = mark;
107         nMarkers--;
108 //System.out.println("Rewinding to " + mark);
109 //try { for (int i = 1; i <= 2; i++) { System.out.println("LA("+i+")=="+LT(i).getText()); } } catch (ScannerException e) {}
110 }
111
112     /** Sync up deferred consumption */
113     private final void syncConsume() {
114         while (numToConsume > 0) {
115             if (nMarkers > 0) {
116                 // guess mode -- leave leading tokens and bump offset.
117 markerOffset++;
118             }
119             else {
120                 // normal mode -- remove first token
121 queue.removeFirst();
122             }
123             numToConsume--;
124         }
125     }
126 }
127
Popular Tags