TokenFilter


1   /*
2    *  Licensed to the Apache Software Foundation (ASF) under one or more
3    *  contributor license agreements.  See the NOTICE file distributed with
4    *  this work for additional information regarding copyright ownership.
5    *  The ASF licenses this file to You under the Apache License, Version 2.0
6    *  (the "License"); you may not use this file except in compliance with
7    *  the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   *  Unless required by applicable law or agreed to in writing, software
12   *  distributed under the License is distributed on an "AS IS" BASIS,
13   *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   *  See the License for the specific language governing permissions and
15   *  limitations under the License.
16   *
17   */
18  package org.apache.tools.ant.filters;
19  
20  import java.io.IOException  ;
21  import java.io.Reader  ;
22  import java.util.Vector  ;
23  import java.util.Enumeration  ;
24  import org.apache.tools.ant.BuildException;
25  import org.apache.tools.ant.ProjectComponent;
26  import org.apache.tools.ant.types.RegularExpression;
27  import org.apache.tools.ant.types.Substitution;
28  import org.apache.tools.ant.util.Tokenizer;
29  import org.apache.tools.ant.util.LineTokenizer;
30  import org.apache.tools.ant.util.StringUtils;
31  import org.apache.tools.ant.util.regexp.Regexp;
32  
33  /**
34   * This splits up input into tokens and passes
35   * the tokens to a sequence of filters.
36   *
37   * @since Ant 1.6
38   * @see BaseFilterReader
39   * @see ChainableReader
40   * @see org.apache.tools.ant.DynamicConfigurator
41   */
42  public class TokenFilter extends BaseFilterReader
43      implements ChainableReader {
44      /**
45       * string filters implement this interface
46       */
47      public interface Filter {
48          /**
49           * filter and/of modify a string
50           *
51           * @param string the string to filter
52           * @return the modified string or null if the
53           *         string did not pass the filter
54           */
55          String   filter(String   string);
56      }
57  
58  
59      /** string filters */
60      private Vector      filters   = new Vector  ();
61      /** the tokenizer to use on the input stream */
62      private Tokenizer tokenizer = null;
63      /** the output token termination */
64      private String      delimOutput = null;
65      /** the current string token from the input stream */
66      private String      line      = null;
67      /** the position in the current string token */
68      private int       linePos   = 0;
69  
70      /**
71       * Constructor for "dummy" instances.
72       *
73       * @see BaseFilterReader#BaseFilterReader()
74       */
75      public TokenFilter() {
76          super();
77      }
78  
79      /**
80       * Creates a new filtered reader.
81       *
82       * @param in A Reader object providing the underlying stream.
83       *           Must not be <code>null</code>.
84       */
85      public TokenFilter(final Reader   in) {
86          super(in);
87      }
88  
89  
90      /**
91       * Returns the next character in the filtered stream, only including
92       * lines from the original stream which match all of the specified
93       * regular expressions.
94       *
95       * @return the next character in the resulting stream, or -1
96       * if the end of the resulting stream has been reached
97       *
98       * @exception IOException if the underlying stream throws an IOException
99       * during reading
100      */
101 
102     public int read() throws IOException   {
103         if (tokenizer == null) {
104             tokenizer = new LineTokenizer();
105         }
106         while (line == null || line.length() == 0) {
107             line = tokenizer.getToken(in);
108             if (line == null) {
109                 return -1;
110             }
111             for (Enumeration   e = filters.elements(); e.hasMoreElements();) {
112                 Filter filter = (Filter) e.nextElement();
113                 line = filter.filter(line);
114                 if (line == null) {
115                     break;
116                 }
117             }
118             linePos = 0;
119             if (line != null) {
120                 if (tokenizer.getPostToken().length() != 0) {
121                     if (delimOutput != null) {
122                         line = line + delimOutput;
123                     } else {
124                         line = line + tokenizer.getPostToken();
125                     }
126                 }
127             }
128         }
129         int ch = line.charAt(linePos);
130         linePos++;
131         if (linePos == line.length()) {
132             line = null;
133         }
134         return ch;
135     }
136 
137     /**
138      * Creates a new TokenFilter using the passed in
139      * Reader for instantiation.
140      *
141      * @param reader A Reader object providing the underlying stream.
142      *
143      * @return a new filter based on this configuration
144      */
145 
146     public final Reader   chain(final Reader   reader) {
147         TokenFilter newFilter = new TokenFilter(reader);
148         newFilter.filters = filters;
149         newFilter.tokenizer = tokenizer;
150         newFilter.delimOutput = delimOutput;
151         newFilter.setProject(getProject());
152         return newFilter;
153     }
154 
155     /**
156      * set the output delimiter.
157      * @param delimOutput replaces the delim string returned by the
158      *                    tokenizer, if present.
159      */
160 
161     public void setDelimOutput(String   delimOutput) {
162         this.delimOutput = resolveBackSlash(delimOutput);
163     }
164 
165     // -----------------------------------------
166     //  Predefined tokenizers
167     // -----------------------------------------
168 
169     /**
170      * add a line tokenizer - this is the default.
171      * @param tokenizer the line tokenizer
172      */
173 
174     public void addLineTokenizer(LineTokenizer tokenizer) {
175         add(tokenizer);
176     }
177 
178     /**
179      * add a string tokenizer
180      * @param tokenizer the string tokenizer
181      */
182 
183     public void addStringTokenizer(StringTokenizer tokenizer) {
184         add(tokenizer);
185     }
186 
187     /**
188      * add a file tokenizer
189      * @param tokenizer the file tokenizer
190      */
191     public void addFileTokenizer(FileTokenizer tokenizer) {
192         add(tokenizer);
193     }
194 
195     /**
196      * add an arbitrary tokenizer
197      * @param tokenizer the tokenizer to all, only one allowed
198      */
199 
200     public void add(Tokenizer tokenizer) {
201         if (this.tokenizer != null) {
202             throw new BuildException("Only one tokenizer allowed");
203         }
204         this.tokenizer = tokenizer;
205     }
206 
207     // -----------------------------------------
208     //  Predefined filters
209     // -----------------------------------------
210 
211     /**
212      * replace string filter
213      * @param filter the replace string filter
214      */
215     public void addReplaceString(ReplaceString filter) {
216         filters.addElement(filter);
217     }
218 
219     /**
220      * contains string filter
221      * @param filter the contains string filter
222      */
223     public void addContainsString(ContainsString filter) {
224         filters.addElement(filter);
225     }
226 
227     /**
228      * replace regex filter
229      * @param filter the replace regex filter
230      */
231     public void addReplaceRegex(ReplaceRegex filter) {
232         filters.addElement(filter);
233     }
234 
235     /**
236      * contains regex filter
237      * @param filter the contains regex filter
238      */
239     public void addContainsRegex(ContainsRegex filter) {
240         filters.addElement(filter);
241     }
242 
243     /**
244      * trim filter
245      * @param filter the trim filter
246      */
247     public void addTrim(Trim filter) {
248         filters.addElement(filter);
249     }
250 
251     /**
252      * ignore blank filter
253      * @param filter the ignore blank filter
254      */
255     public void addIgnoreBlank(IgnoreBlank filter) {
256         filters.addElement(filter);
257     }
258 
259     /**
260      * delete chars
261      * @param filter the delete characters filter
262      */
263     public void addDeleteCharacters(DeleteCharacters filter) {
264         filters.addElement(filter);
265     }
266 
267     /**
268      * Add an arbitrary filter
269      * @param filter the filter to add
270      */
271     public void add(Filter filter) {
272         filters.addElement(filter);
273     }
274 
275 
276     // --------------------------------------------
277     //
278     //      Tokenizer Classes (impls moved to oata.util)
279     //
280     // --------------------------------------------
281 
282     /**
283      * class to read the complete input into a string
284      */
285     public static class FileTokenizer
286         extends org.apache.tools.ant.util.FileTokenizer {
287     }
288 
289     /**
290      * class to tokenize the input as areas separated
291      * by white space, or by a specified list of
292      * delim characters. Behaves like java.util.StringTokenizer.
293      * if the stream starts with delim characters, the first
294      * token will be an empty string (unless the treat delims
295      * as tokens flag is set).
296      */
297     public static class StringTokenizer
298         extends org.apache.tools.ant.util.StringTokenizer {
299     }
300 
301     // --------------------------------------------
302     //
303     //      Filter classes
304     //
305     // --------------------------------------------
306 
307     /**
308      * Abstract class that converts derived filter classes into
309      * ChainableReaderFilter's
310      */
311     public abstract static class ChainableReaderFilter extends ProjectComponent
312         implements ChainableReader, Filter {
313         private boolean byLine = true;
314 
315         /**
316          * set whether to use filetokenizer or line tokenizer
317          * @param byLine if true use a linetokenizer (default) otherwise
318          *               use a filetokenizer
319          */
320         public void setByLine(boolean byLine) {
321             this.byLine = byLine;
322         }
323 
324         /**
325          * Chain a tokenfilter reader to a reader,
326          *
327          * @param reader the input reader object
328          * @return the chained reader object
329          */
330         public Reader   chain(Reader   reader) {
331             TokenFilter tokenFilter = new TokenFilter(reader);
332             if (!byLine) {
333                 tokenFilter.add(new FileTokenizer());
334             }
335             tokenFilter.add(this);
336             return tokenFilter;
337         }
338     }
339 
340     /**
341      * Simple replace string filter.
342      */
343     public static class ReplaceString extends ChainableReaderFilter {
344         private String   from;
345         private String   to;
346 
347         /**
348          * the from attribute
349          *
350          * @param from the string to replace
351          */
352         public void setFrom(String   from) {
353             this.from = from;
354         }
355 
356         /**
357          * the to attribute
358          *
359          * @param to the string to replace 'from' with
360          */
361         public void setTo(String   to) {
362             this.to = to;
363         }
364 
365         /**
366          * Filter a string 'line' replacing from with to
367          * (C&P from the Replace task)
368          * @param line the string to be filtered
369          * @return the filtered line
370          */
371         public String   filter(String   line) {
372             if (from == null) {
373                 throw new BuildException("Missing from in stringreplace");
374             }
375             StringBuffer   ret = new StringBuffer  ();
376             int start = 0;
377             int found = line.indexOf(from);
378             while (found >= 0) {
379                 // write everything up to the from
380                 if (found > start) {
381                     ret.append(line.substring(start, found));
382                 }
383 
384                 // write the replacement to
385                 if (to != null) {
386                     ret.append(to);
387                 }
388 
389                 // search again
390                 start = found + from.length();
391                 found = line.indexOf(from, start);
392             }
393 
394             // write the remaining characters
395             if (line.length() > start) {
396                 ret.append(line.substring(start, line.length()));
397             }
398 
399             return ret.toString();
400         }
401     }
402 
403     /**
404      * Simple filter to filter lines contains strings
405      */
406     public static class ContainsString extends ProjectComponent
407         implements Filter {
408         private String   contains;
409 
410         /**
411          * the contains attribute
412          * @param contains the string that the token should contain
413          */
414         public void setContains(String   contains) {
415             this.contains = contains;
416         }
417 
418         /**
419          * Filter strings that contain the contains attribute
420          *
421          * @param string the string to be filtered
422          * @return null if the string does not contain "contains",
423          *              string otherwise
424          */
425         public String   filter(String   string) {
426             if (contains == null) {
427                 throw new BuildException("Missing contains in containsstring");
428             }
429             if (string.indexOf(contains) > -1) {
430                 return string;
431             }
432             return null;
433         }
434     }
435 
436     /**
437      * filter to replace regex.
438      */
439     public static class ReplaceRegex extends ChainableReaderFilter {
440         private String               from;
441         private String               to;
442         private RegularExpression  regularExpression;
443         private Substitution       substitution;
444         private boolean            initialized = false;
445         private String               flags = "";
446         private int                options;
447         private Regexp             regexp;
448 
449         /**
450          * the from attribute
451          * @param from the regex string
452          */
453         public void setPattern(String   from) {
454             this.from = from;
455         }
456         /**
457          * the to attribute
458          * @param to the replacement string
459          */
460         public void setReplace(String   to) {
461             this.to = to;
462         }
463 
464         /**
465          * @param flags the regex flags
466          */
467         public void setFlags(String   flags) {
468             this.flags = flags;
469         }
470 
471         private void initialize() {
472             if (initialized) {
473                 return;
474             }
475             options = convertRegexOptions(flags);
476             if (from == null) {
477                 throw new BuildException("Missing pattern in replaceregex");
478             }
479             regularExpression = new RegularExpression();
480             regularExpression.setPattern(from);
481             regexp = regularExpression.getRegexp(getProject());
482             if (to == null) {
483                 to = "";
484             }
485             substitution = new Substitution();
486             substitution.setExpression(to);
487         }
488 
489         /**
490          * @param line the string to modify
491          * @return the modified string
492          */
493         public String   filter(String   line) {
494             initialize();
495 
496             if (!regexp.matches(line, options)) {
497                 return line;
498             }
499             return regexp.substitute(
500                 line, substitution.getExpression(getProject()), options);
501         }
502     }
503 
504     /**
505      * filter to filter tokens matching regular expressions.
506      */
507     public static class ContainsRegex extends ChainableReaderFilter {
508         private String               from;
509         private String               to;
510         private RegularExpression  regularExpression;
511         private Substitution       substitution;
512         private boolean            initialized = false;
513         private String               flags = "";
514         private int                options;
515         private Regexp             regexp;
516 
517 
518         /**
519          * @param from the regex pattern
520          */
521         public void setPattern(String   from) {
522             this.from = from;
523         }
524 
525         /**
526          * @param to the replacement string
527          */
528         public void setReplace(String   to) {
529             this.to = to;
530         }
531 
532         /**
533          * @param flags the regex flags
534          */
535         public void setFlags(String   flags) {
536             this.flags = flags;
537         }
538 
539         private void initialize() {
540             if (initialized) {
541                 return;
542             }
543             options = convertRegexOptions(flags);
544             if (from == null) {
545                 throw new BuildException("Missing from in containsregex");
546             }
547             regularExpression = new RegularExpression();
548             regularExpression.setPattern(from);
549             regexp = regularExpression.getRegexp(getProject());
550             if (to == null) {
551                 return;
552             }
553             substitution = new Substitution();
554             substitution.setExpression(to);
555         }
556 
557         /**
558          * apply regex and substitution on a string
559          * @param string the string to apply filter on
560          * @return the filtered string
561          */
562         public String   filter(String   string) {
563             initialize();
564             if (!regexp.matches(string, options)) {
565                 return null;
566             }
567             if (substitution == null) {
568                 return string;
569             }
570             return regexp.substitute(
571                 string, substitution.getExpression(getProject()), options);
572         }
573     }
574 
575     /** Filter to trim white space */
576     public static class Trim extends ChainableReaderFilter {
577         /**
578          * @param line the string to be trimmed
579          * @return the trimmed string
580          */
581         public String   filter(String   line) {
582             return line.trim();
583         }
584     }
585 
586 
587 
588     /** Filter remove empty tokens */
589     public static class IgnoreBlank extends ChainableReaderFilter {
590         /**
591          * @param line the line to modify
592          * @return the trimmed line
593          */
594         public String   filter(String   line) {
595             if (line.trim().length() == 0) {
596                 return null;
597             }
598             return line;
599         }
600     }
601 
602     /**
603      * Filter to delete characters
604      */
605     public static class DeleteCharacters extends ProjectComponent
606         implements Filter, ChainableReader {
607         // Attributes
608         /** the list of characters to remove from the input */
609         private String   deleteChars = "";
610 
611         /**
612          * Set the list of characters to delete
613          * @param deleteChars the list of characters
614          */
615         public void setChars(String   deleteChars) {
616             this.deleteChars = resolveBackSlash(deleteChars);
617         }
618 
619         /**
620          * remove characters from a string
621          * @param string the string to remove the characters from
622          * @return the converted string
623          */
624         public String   filter(String   string) {
625             StringBuffer   output = new StringBuffer  (string.length());
626             for (int i = 0; i < string.length(); ++i) {
627                 char ch = string.charAt(i);
628                 if (!(isDeleteCharacter(ch))) {
629                     output.append(ch);
630                 }
631             }
632             return output.toString();
633         }
634 
635         /**
636          * factory method to provide a reader that removes
637          * the characters from a reader as part of a filter
638          * chain
639          * @param reader the reader object
640          * @return the chained reader object
641          */
642         public Reader   chain(Reader   reader) {
643             return new BaseFilterReader(reader) {
644                 /**
645                  * @return the next non delete character
646                  */
647                 public int read()
648                     throws IOException   {
649                     while (true) {
650                         int c = in.read();
651                         if (c == -1) {
652                             return c;
653                         }
654                         if (!(isDeleteCharacter((char) c))) {
655                             return c;
656                         }
657                     }
658                 }
659             };
660         }
661 
662         /** check if the character c is to be deleted */
663         private boolean isDeleteCharacter(char c) {
664             for (int d = 0; d < deleteChars.length(); ++d) {
665                 if (deleteChars.charAt(d) ==  c) {
666                     return true;
667                 }
668             }
669             return false;
670         }
671     }
672 
673     // --------------------------------------------------------
674     //  static utility methods - could be placed somewhere else
675     // --------------------------------------------------------
676 
677     /**
678      * xml does not do "c" like interpretation of strings.
679      * i.e. \n\r\t etc.
680      * this method processes \n, \r, \t, \f, \\
681      * also subs \s -> " \n\r\t\f"
682      * a trailing '\' will be ignored
683      *
684      * @param input raw string with possible embedded '\'s
685      * @return converted string
686      */
687     public static String   resolveBackSlash(String   input) {
688         return StringUtils.resolveBackSlash(input);
689     }
690 
691     /**
692      * convert regex option flag characters to regex options
693      * <dl>
694      *   <li>g -  Regexp.REPLACE_ALL</li>
695      *   <li>i -  Regexp.MATCH_CASE_INSENSITIVE</li>
696      *   <li>m -  Regexp.MATCH_MULTILINE</li>
697      *   <li>s -  Regexp.MATCH_SINGLELINE</li>
698      * </dl>
699      * @param flags the string containing the flags
700      * @return the Regexp option bits
701      */
702     public static int convertRegexOptions(String   flags) {
703         if (flags == null) {
704             return 0;
705         }
706         int options = 0;
707         if (flags.indexOf('g') != -1) {
708             options |= Regexp.REPLACE_ALL;
709         }
710         if (flags.indexOf('i') != -1) {
711             options |= Regexp.MATCH_CASE_INSENSITIVE;
712         }
713         if (flags.indexOf('m') != -1) {
714             options |= Regexp.MATCH_MULTILINE;
715         }
716         if (flags.indexOf('s') != -1) {
717             options |= Regexp.MATCH_SINGLELINE;
718         }
719         return options;
720     }
721 }
722
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags