TokenStreamRewriteEngine


1   package persistence.antlr;
2   
3   /* ANTLR Translator Generator
4    * Project led by Terence Parr at http://www.jGuru.com
5    * Software rights: http://www.antlr.org/license.html
6    */
7   
8   import persistence.antlr.collections.impl.BitSet;
9   import java.util.*;
10  
11  /** This token stream tracks the *entire* token stream coming from
12   *  a lexer, but does not pass on the whitespace (or whatever else
13   *  you want to discard) to the parser.
14   *
15   *  This class can then be asked for the ith token in the input stream.
16   *  Useful for dumping out the input stream exactly after doing some
17   *  augmentation or other manipulations.  Tokens are index from 0..n-1
18   *
19   *  You can insert stuff, replace, and delete chunks.  Note that the
20   *  operations are done lazily--only if you convert the buffer to a
21   *  String.  This is very efficient because you are not moving data around
22   *  all the time.  As the buffer of tokens is converted to strings, the
23   *  toString() method(s) check to see if there is an operation at the
24   *  current index.  If so, the operation is done and then normal String
25   *  rendering continues on the buffer.  This is like having multiple Turing
26   *  machine instruction streams (programs) operating on a single input tape. :)
27   *
28   *  Since the operations are done lazily at toString-time, operations do not
29   *  screw up the token index values.  That is, an insert operation at token
30   *  index i does not change the index values for tokens i+1..n-1.
31   *
32   *  Because operations never actually alter the buffer, you may always get
33   *  the original token stream back without undoing anything.  Since
34   *  the instructions are queued up, you can easily simulate transactions and
35   *  roll back any changes if there is an error just by removing instructions.
36   *  For example,
37   *
38   *      TokenStreamRewriteEngine rewriteEngine =
39   *          new TokenStreamRewriteEngine(lexer);
40   *      JavaRecognizer parser = new JavaRecognizer(rewriteEngine);
41   *      ...
42   *      rewriteEngine.insertAfter("pass1", t, "foobar");}
43   *      rewriteEngine.insertAfter("pass2", u, "start");}
44   *      System.out.println(rewriteEngine.toString("pass1"));
45   *      System.out.println(rewriteEngine.toString("pass2"));
46   *
47   *  You can also have multiple "instruction streams" and get multiple
48   *  rewrites from a single pass over the input.  Just name the instruction
49   *  streams and use that name again when printing the buffer.  This could be
50   *  useful for generating a C file and also its header file--all from the
51   *  same buffer.
52   *
53   *  If you don't use named rewrite streams, a "default" stream is used.
54   *
55   *  Terence Parr, parrt@cs.usfca.edu
56   *  University of San Francisco
57   *  February 2004
58   */
59  public class TokenStreamRewriteEngine implements TokenStream {
60      public static final int MIN_TOKEN_INDEX = 0;
61  
62      static class RewriteOperation {
63          protected int index;
64          protected String   text;
65          protected RewriteOperation(int index, String   text) {
66              this.index = index;
67              this.text = text;
68          }
69          /** Execute the rewrite operation by possibly adding to the buffer.
70           *  Return the index of the next token to operate on.
71           */
72          public int execute(StringBuffer   buf) {
73              return index;
74          }
75      }
76  
77      static class InsertBeforeOp extends RewriteOperation {
78          public InsertBeforeOp(int index, String   text) {
79              super(index,text);
80          }
81          public int execute(StringBuffer   buf) {
82              buf.append(text);
83              return index;
84          }
85      }
86  
87      static class ReplaceOp extends RewriteOperation {
88          protected int lastIndex;
89          public ReplaceOp(int from, int to, String   text) {
90              super(from,text);
91              lastIndex = to;
92          }
93          public int execute(StringBuffer   buf) {
94              if ( text!=null ) {
95                  buf.append(text);
96              }
97              return lastIndex+1;
98          }
99      }
100 
101     static class DeleteOp extends ReplaceOp {
102         public DeleteOp(int from, int to) {
103             super(from, to, null);
104         }
105     }
106 
107     public static final String   DEFAULT_PROGRAM_NAME = "default";
108     public static final int PROGRAM_INIT_SIZE = 100;
109 
110     /** Track the incoming list of tokens */
111     protected List tokens;
112 
113     /** You may have multiple, named streams of rewrite operations.
114      *  I'm calling these things "programs."
115      *  Maps String (name) -> rewrite (List)
116      */
117     protected Map programs = null;
118 
119     /** Map String (program name) -> Integer index */
120     protected Map lastRewriteTokenIndexes = null;
121 
122     /** track index of tokens */
123     protected int index = MIN_TOKEN_INDEX;
124 
125     /** Who do we suck tokens from? */
126     protected TokenStream stream;
127 
128     /** Which (whitespace) token(s) to throw out */
129     protected BitSet discardMask = new BitSet();
130 
131     public TokenStreamRewriteEngine(TokenStream upstream) {
132         this(upstream,1000);
133     }
134 
135     public TokenStreamRewriteEngine(TokenStream upstream, int initialSize) {
136         stream = upstream;
137         tokens = new ArrayList(initialSize);
138         programs = new HashMap();
139         programs.put(DEFAULT_PROGRAM_NAME,
140                                new ArrayList(PROGRAM_INIT_SIZE));
141         lastRewriteTokenIndexes = new HashMap();
142     }
143 
144     public Token nextToken() throws TokenStreamException {
145         TokenWithIndex t;
146         // suck tokens until end of stream or we find a non-discarded token
147         do {
148             t = (TokenWithIndex)stream.nextToken();
149             if ( t!=null ) {
150                 t.setIndex(index);  // what is t's index in list?
151                 if ( t.getType()!=Token.EOF_TYPE ) {
152                     tokens.add(t);  // track all tokens except EOF
153                 }
154                 index++;            // move to next position
155             }
156         } while ( t!=null && discardMask.member(t.getType()) );
157         return t;
158     }
159 
160     public void rollback(int instructionIndex) {
161         rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
162     }
163 
164     /** Rollback the instruction stream for a program so that
165      *  the indicated instruction (via instructionIndex) is no
166      *  longer in the stream.  UNTESTED!
167      */
168     public void rollback(String   programName, int instructionIndex) {
169         List is = (List)programs.get(programName);
170         if ( is!=null ) {
171             programs.put(programName, is.subList(MIN_TOKEN_INDEX,instructionIndex));
172         }
173     }
174 
175     public void deleteProgram() {
176         deleteProgram(DEFAULT_PROGRAM_NAME);
177     }
178 
179     /** Reset the program so that no instructions exist */
180     public void deleteProgram(String   programName) {
181         rollback(programName, MIN_TOKEN_INDEX);
182     }
183 
184     /** If op.index > lastRewriteTokenIndexes, just add to the end.
185      *  Otherwise, do linear */
186     protected void addToSortedRewriteList(RewriteOperation op) {
187         addToSortedRewriteList(DEFAULT_PROGRAM_NAME, op);
188     }
189 
190     protected void addToSortedRewriteList(String   programName, RewriteOperation op) {
191         List rewrites = getProgram(programName);
192         // if at or beyond last op's index, just append
193         if ( op.index>=getLastRewriteTokenIndex(programName) ) {
194             rewrites.add(op); // append to list of operations
195             // record the index of this operation for next time through
196             setLastRewriteTokenIndex(programName, op.index);
197             return;
198         }
199         // not after the last one, so must insert to ordered list
200         Comparator comparator = new Comparator() {
201             public int compare(Object   o, Object   o1) {
202                 RewriteOperation a = (RewriteOperation)o;
203                 RewriteOperation b = (RewriteOperation)o1;
204                 if ( a.index<b.index ) return -1;
205                 if ( a.index>b.index ) return 1;
206                 return 0;
207             }
208         };
209         int pos = Collections.binarySearch(rewrites, op, comparator);
210         if ( pos<0 ) {
211             rewrites.add(-pos-1, op);
212         }
213     }
214 
215     public void insertAfter(Token t, String   text) {
216         insertAfter(DEFAULT_PROGRAM_NAME, t, text);
217     }
218 
219     public void insertAfter(int index, String   text) {
220         insertAfter(DEFAULT_PROGRAM_NAME, index, text);
221     }
222 
223     public void insertAfter(String   programName, Token t, String   text) {
224         insertAfter(programName,((TokenWithIndex)t).getIndex(), text); 
225     }
226 
227     public void insertAfter(String   programName, int index, String   text) {
228         // to insert after, just insert before next index (even if past end)
229         insertBefore(programName,index+1, text); 
230     }
231 
232     public void insertBefore(Token t, String   text) {
233         insertBefore(DEFAULT_PROGRAM_NAME, t, text);
234     }
235 
236     public void insertBefore(int index, String   text) {
237         insertBefore(DEFAULT_PROGRAM_NAME, index, text);
238     }
239 
240     public void insertBefore(String   programName, Token t, String   text) {
241         insertBefore(programName, ((TokenWithIndex)t).getIndex(), text);
242     }
243 
244     public void insertBefore(String   programName, int index, String   text) {
245         addToSortedRewriteList(programName, new InsertBeforeOp(index,text));
246     }
247 
248     public void replace(int index, String   text) {
249         replace(DEFAULT_PROGRAM_NAME, index, index, text);
250     }
251 
252     public void replace(int from, int to, String   text) {
253         replace(DEFAULT_PROGRAM_NAME, from, to, text);
254     }
255 
256     public void replace(Token indexT, String   text) {
257         replace(DEFAULT_PROGRAM_NAME, indexT, indexT, text);
258     }
259 
260     public void replace(Token from, Token to, String   text) {
261         replace(DEFAULT_PROGRAM_NAME, from, to, text);
262     }
263 
264     public void replace(String   programName, int from, int to, String   text) {
265         addToSortedRewriteList(new ReplaceOp(from, to, text));
266     }
267 
268     public void replace(String   programName, Token from, Token to, String   text) {
269         replace(programName,
270                 ((TokenWithIndex)from).getIndex(),
271                 ((TokenWithIndex)to).getIndex(),
272                 text);
273     }
274 
275     public void delete(int index) {
276         delete(DEFAULT_PROGRAM_NAME, index, index);
277     }
278 
279     public void delete(int from, int to) {
280         delete(DEFAULT_PROGRAM_NAME, from, to);
281     }
282 
283     public void delete(Token indexT) {
284         delete(DEFAULT_PROGRAM_NAME, indexT, indexT);
285     }
286 
287     public void delete(Token from, Token to) {
288         delete(DEFAULT_PROGRAM_NAME, from, to);
289     }
290 
291     public void delete(String   programName, int from, int to) {
292         replace(programName,from,to,null);
293     }
294 
295     public void delete(String   programName, Token from, Token to) {
296         replace(programName,from,to,null);
297     }
298 
299     public void discard(int ttype) {
300         discardMask.add(ttype);
301     }
302 
303     public TokenWithIndex getToken(int i) {
304         return (TokenWithIndex)tokens.get(i);
305     }
306 
307     public int getTokenStreamSize() {
308         return tokens.size();
309     }
310 
311     public String   toOriginalString() {
312         return toOriginalString(MIN_TOKEN_INDEX, getTokenStreamSize()-1);
313     }
314 
315     public String   toOriginalString(int start, int end) {
316         StringBuffer   buf = new StringBuffer  ();
317         for (int i=start; i>=MIN_TOKEN_INDEX && i<=end && i<tokens.size(); i++) {
318             buf.append(getToken(i).getText());
319         }
320         return buf.toString();
321     }
322 
323     public String   toString() {
324         return toString(MIN_TOKEN_INDEX, getTokenStreamSize());
325     }
326 
327     public String   toString(String   programName) {
328         return toString(programName, MIN_TOKEN_INDEX, getTokenStreamSize());
329     }
330 
331     public String   toString(int start, int end) {
332         return toString(DEFAULT_PROGRAM_NAME, start, end);
333     }
334 
335     public String   toString(String   programName, int start, int end) {
336         List rewrites = (List)programs.get(programName);
337         if ( rewrites==null ) {
338             return null; // invalid program
339         }
340         StringBuffer   buf = new StringBuffer  ();
341 
342         /** Index of first rewrite we have not done */
343         int rewriteOpIndex = 0;
344 
345         int tokenCursor=start;
346         while ( tokenCursor>=MIN_TOKEN_INDEX &&
347                 tokenCursor<=end &&
348                 tokenCursor<tokens.size() )
349         {
350             if ( rewriteOpIndex<rewrites.size() ) {
351                 RewriteOperation op =
352                         (RewriteOperation)rewrites.get(rewriteOpIndex);
353                 while ( tokenCursor==op.index && rewriteOpIndex<rewrites.size() ) {
354                     /*
355                     System.out.println("execute op "+rewriteOpIndex+
356                                        " (type "+op.getClass().getName()+")"
357                                        +" at index "+op.index);
358                     */
359                     tokenCursor = op.execute(buf);
360                     rewriteOpIndex++;
361                     if ( rewriteOpIndex<rewrites.size() ) {
362                         op = (RewriteOperation)rewrites.get(rewriteOpIndex);
363                     }
364                 }
365             }
366             if ( tokenCursor<end ) {
367                 buf.append(getToken(tokenCursor).getText());
368                 tokenCursor++;
369             }
370         }
371         // now see if there are operations (append) beyond last token index
372         for (int opi=rewriteOpIndex; opi<rewrites.size(); opi++) {
373             RewriteOperation op =
374                     (RewriteOperation)rewrites.get(opi);
375             op.execute(buf); // must be insertions if after last token
376         }
377 
378         return buf.toString();
379     }
380 
381     public String   toDebugString() {
382         return toDebugString(MIN_TOKEN_INDEX, getTokenStreamSize());
383     }
384 
385     public String   toDebugString(int start, int end) {
386         StringBuffer   buf = new StringBuffer  ();
387         for (int i=start; i>=MIN_TOKEN_INDEX && i<=end && i<tokens.size(); i++) {
388             buf.append(getToken(i));
389         }
390         return buf.toString();
391     }
392 
393     public int getLastRewriteTokenIndex() {
394         return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
395     }
396 
397     protected int getLastRewriteTokenIndex(String   programName) {
398         Integer   I = (Integer  )lastRewriteTokenIndexes.get(programName);
399         if ( I==null ) {
400             return -1;
401         }
402         return I.intValue();
403     }
404 
405     protected void setLastRewriteTokenIndex(String   programName, int i) {
406         lastRewriteTokenIndexes.put(programName, new Integer  (i));
407     }
408 
409     protected List getProgram(String   name) {
410         List is = (List)programs.get(name);
411         if ( is==null ) {
412             is = initializeProgram(name);
413         }
414         return is;
415     }
416 
417     private List initializeProgram(String   name) {
418         List is = new ArrayList(PROGRAM_INIT_SIZE);
419         programs.put(name, is);
420         return is;
421     }
422 }
423
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags