KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > antlr > TokenStreamHiddenTokenFilter


1 package antlr;
2
3 /* ANTLR Translator Generator
4  * Project led by Terence Parr at http://www.jGuru.com
5  * Software rights: http://www.antlr.org/RIGHTS.html
6  *
7  * $Id: //depot/code/org.antlr/main/main/antlr/TokenStreamHiddenTokenFilter.java#4 $
8  */

9
10 import antlr.collections.impl.BitSet;
11
12 /**This object filters a token stream coming from a lexer
13  * or another TokenStream so that only certain token channels
14  * get transmitted to the parser.
15  *
16  * Any of the channels can be filtered off as "hidden" channels whose
17  * tokens can be accessed from the parser.
18  */

19 public class TokenStreamHiddenTokenFilter extends TokenStreamBasicFilter implements TokenStream {
20     // protected BitSet discardMask;
21
protected BitSet hideMask;
22
23     private CommonHiddenStreamToken nextMonitoredToken;
24
25     /** track tail of hidden list emanating from previous
26      * monitored token
27      */

28     protected CommonHiddenStreamToken lastHiddenToken;
29
30     protected CommonHiddenStreamToken firstHidden = null;
31
32     public TokenStreamHiddenTokenFilter(TokenStream input) {
33         super(input);
34         hideMask = new BitSet();
35     }
36
37     protected void consume() throws TokenStreamException {
38         nextMonitoredToken = (CommonHiddenStreamToken)input.nextToken();
39     }
40
41     private void consumeFirst() throws TokenStreamException {
42         consume(); // get first token of input stream
43

44         // Handle situation where hidden or discarded tokens
45
// appear first in input stream
46
CommonHiddenStreamToken p = null;
47         // while hidden or discarded scarf tokens
48
while (hideMask.member(LA(1).getType()) || discardMask.member(LA(1).getType())) {
49             if (hideMask.member(LA(1).getType())) {
50                 if (p == null) {
51                     p = LA(1);
52                 }
53                 else {
54                     p.setHiddenAfter(LA(1));
55                     LA(1).setHiddenBefore(p); // double-link
56
p = LA(1);
57                 }
58                 lastHiddenToken = p;
59                 if (firstHidden == null) {
60                     firstHidden = p; // record hidden token if first
61
}
62             }
63             consume();
64         }
65     }
66
67     public BitSet getDiscardMask() {
68         return discardMask;
69     }
70
71     /** Return a ptr to the hidden token appearing immediately after
72      * token t in the input stream.
73      */

74     public CommonHiddenStreamToken getHiddenAfter(CommonHiddenStreamToken t) {
75         return t.getHiddenAfter();
76     }
77
78     /** Return a ptr to the hidden token appearing immediately before
79      * token t in the input stream.
80      */

81     public CommonHiddenStreamToken getHiddenBefore(CommonHiddenStreamToken t) {
82         return t.getHiddenBefore();
83     }
84
85     public BitSet getHideMask() {
86         return hideMask;
87     }
88
89     /** Return the first hidden token if one appears
90      * before any monitored token.
91      */

92     public CommonHiddenStreamToken getInitialHiddenToken() {
93         return firstHidden;
94     }
95
96     public void hide(int m) {
97         hideMask.add(m);
98     }
99
100     public void hide(BitSet mask) {
101         hideMask = mask;
102     }
103
104     protected CommonHiddenStreamToken LA(int i) {
105         return nextMonitoredToken;
106     }
107
108     /** Return the next monitored token.
109      * Test the token following the monitored token.
110      * If following is another monitored token, save it
111      * for the next invocation of nextToken (like a single
112      * lookahead token) and return it then.
113      * If following is unmonitored, nondiscarded (hidden)
114      * channel token, add it to the monitored token.
115      *
116      * Note: EOF must be a monitored Token.
117      */

118     public Token nextToken() throws TokenStreamException {
119         // handle an initial condition; don't want to get lookahead
120
// token of this splitter until first call to nextToken
121
if (LA(1) == null) {
122             consumeFirst();
123         }
124
125         // we always consume hidden tokens after monitored, thus,
126
// upon entry LA(1) is a monitored token.
127
CommonHiddenStreamToken monitored = LA(1);
128         // point to hidden tokens found during last invocation
129
monitored.setHiddenBefore(lastHiddenToken);
130         lastHiddenToken = null;
131
132         // Look for hidden tokens, hook them into list emanating
133
// from the monitored tokens.
134
consume();
135         CommonHiddenStreamToken p = monitored;
136         // while hidden or discarded scarf tokens
137
while (hideMask.member(LA(1).getType()) || discardMask.member(LA(1).getType())) {
138             if (hideMask.member(LA(1).getType())) {
139                 // attach the hidden token to the monitored in a chain
140
// link forwards
141
p.setHiddenAfter(LA(1));
142                 // link backwards
143
if (p != monitored) { //hidden cannot point to monitored tokens
144
LA(1).setHiddenBefore(p);
145                 }
146                 p = lastHiddenToken = LA(1);
147             }
148             consume();
149         }
150         return monitored;
151     }
152 }
153
Popular Tags