KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > persistence > antlr > TokenBuffer


1 package persistence.antlr;
2
3 /* ANTLR Translator Generator
4  * Project led by Terence Parr at http://www.jGuru.com
5  * Software rights: http://www.antlr.org/license.html
6  *
7  */
8
9 /**A Stream of Token objects fed to the parser from a Tokenizer that can
10  * be rewound via mark()/rewind() methods.
11  * <p>
12  * A dynamic array is used to buffer up all the input tokens. Normally,
13  * "k" tokens are stored in the buffer. More tokens may be stored during
14  * guess mode (testing syntactic predicate), or when LT(i>k) is referenced.
15  * Consumption of tokens is deferred. In other words, reading the next
16  * token is not done by conume(), but deferred until needed by LA or LT.
17  * <p>
18  *
19  * @see persistence.antlr.Token
20  * @see persistence.antlr.Tokenizer
21  * @see persistence.antlr.TokenQueue
22  */
23
24 import java.io.IOException JavaDoc;
25
26 public class TokenBuffer {
27
28     // Token source
29 protected TokenStream input;
30
31     // Number of active markers
32 int nMarkers = 0;
33
34     // Additional offset used when markers are active
35 int markerOffset = 0;
36
37     // Number of calls to consume() since last LA() or LT() call
38 int numToConsume = 0;
39
40     // Circular queue
41 TokenQueue queue;
42
43     /** Create a token buffer */
44     public TokenBuffer(TokenStream input_) {
45         input = input_;
46         queue = new TokenQueue(1);
47     }
48
49     /** Reset the input buffer to empty state */
50     public final void reset() {
51         nMarkers = 0;
52         markerOffset = 0;
53         numToConsume = 0;
54         queue.reset();
55     }
56
57     /** Mark another token for deferred consumption */
58     public final void consume() {
59         numToConsume++;
60     }
61
62     /** Ensure that the token buffer is sufficiently full */
63     private final void fill(int amount) throws TokenStreamException {
64         syncConsume();
65         // Fill the buffer sufficiently to hold needed tokens
66 while (queue.nbrEntries < amount + markerOffset) {
67             // Append the next token
68 queue.append(input.nextToken());
69         }
70     }
71
72     /** return the Tokenizer (needed by ParseView) */
73     public TokenStream getInput() {
74         return input;
75     }
76
77     /** Get a lookahead token value */
78     public final int LA(int i) throws TokenStreamException {
79         fill(i);
80         return queue.elementAt(markerOffset + i - 1).type;
81     }
82
83     /** Get a lookahead token */
84     public final Token LT(int i) throws TokenStreamException {
85         fill(i);
86         return queue.elementAt(markerOffset + i - 1);
87     }
88
89     /**Return an integer marker that can be used to rewind the buffer to
90      * its current state.
91      */
92     public final int mark() {
93         syncConsume();
94 //System.out.println("Marking at " + markerOffset);
95 //try { for (int i = 1; i <= 2; i++) { System.out.println("LA("+i+")=="+LT(i).getText()); } } catch (ScannerException e) {}
96 nMarkers++;
97         return markerOffset;
98     }
99
100     /**Rewind the token buffer to a marker.
101      * @param mark Marker returned previously from mark()
102      */
103     public final void rewind(int mark) {
104         syncConsume();
105         markerOffset = mark;
106         nMarkers--;
107 //System.out.println("Rewinding to " + mark);
108 //try { for (int i = 1; i <= 2; i++) { System.out.println("LA("+i+")=="+LT(i).getText()); } } catch (ScannerException e) {}
109 }
110
111     /** Sync up deferred consumption */
112     private final void syncConsume() {
113         while (numToConsume > 0) {
114             if (nMarkers > 0) {
115                 // guess mode -- leave leading tokens and bump offset.
116 markerOffset++;
117             }
118             else {
119                 // normal mode -- remove first token
120 queue.removeFirst();
121             }
122             numToConsume--;
123         }
124     }
125 }
126
Popular Tags