FilePatternFilter


1   /*
2    * PathSuffixFilter
3    *
4    * $Id: FilePatternFilter.java,v 1.11.2.1 2007/01/13 01:31:20 stack-sf Exp $
5    *
6    * Created on Mar 11, 2004
7    *
8    * Copyright (C) 2004 Internet Archive.
9    *
10   * This file is part of the Heritrix web crawler (crawler.archive.org).
11   *
12   * Heritrix is free software; you can redistribute it and/or modify
13   * it under the terms of the GNU Lesser Public License as published by
14   * the Free Software Foundation; either version 2.1 of the License, or
15   * any later version.
16   *
17   * Heritrix is distributed in the hope that it will be useful,
18   * but WITHOUT ANY WARRANTY; without even the implied warranty of
19   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20   * GNU Lesser Public License for more details.
21   *
22   * You should have received a copy of the GNU Lesser Public License
23   * along with Heritrix; if not, write to the Free Software
24   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
25   */
26  
27  package org.archive.crawler.filter;
28  
29  import java.util.logging.Level  ;
30  import java.util.logging.Logger  ;
31  
32  import javax.management.AttributeNotFoundException  ;
33  
34  import org.archive.crawler.datamodel.CrawlURI;
35  import org.archive.crawler.settings.ComplexType;
36  import org.archive.crawler.settings.MapType;
37  import org.archive.crawler.settings.SimpleType;
38  
39  /**
40   * Compares suffix of a passed CrawlURI, UURI, or String against a regular
41   * expression pattern accepting matches.
42   *
43   * @author Igor Ranitovic
44   * @deprecated As of release 1.10.0.  Replaced by
45   * {@link MatchesFilePatternDecideRule}.
46   */
47  public class FilePatternFilter extends URIRegExpFilter {
48  
49      private static final long serialVersionUID = -4019256104085004651L;
50  
51      private static final Logger   logger =
52          Logger.getLogger(FilePatternFilter.class.getName());
53      public static final String   ATTR_USE_DEFAULT = "use-default-patterns";
54      public static final String   IMAGES_PATTERNS = ".*(?i)(\\.(bmp|gif|jpe?g" +
55          "|png|tiff?))$";
56      public static final String   AUDIO_PATTERNS = ".*(?i)(\\.(mid|mp2|mp3|mp4" +
57          "|wav))$";
58      public static final String   VIDEO_PATTERNS = ".*(?i)(\\.(avi|mov|mpeg|ram" +
59          "|rm|smil|wmv))$";
60      public static final String   MISC_PATTERNS = ".*(?i)(\\.(doc|pdf|ppt|swf))$";
61      public static final String   ALL_DEFAULT_PATTERNS = ".*(?i)(\\.(bmp|gif" +
62          "|jpe?g|png|tiff?|mid|mp2|mp3|mp4|wav|avi|mov|mpeg|ram|rm|smil|wmv" +
63          "|doc|pdf|ppt|swf))$";
64  
65      public static final String   ALL = "All";
66      public static final String   IMAGES = "Images";
67      public static final String   AUDIO = "Audio";
68      public static final String   VIDEO = "Video";
69      public static final String   MISC = "Miscellaneous";
70      public static final String   CUSTOM = "Custom";
71  
72      /**
73       * @param name
74       */
75      public FilePatternFilter(String   name) {
76          super(name);
77          setDescription("A URI path suffix filter *Deprecated* Use" +
78                  "DecidingFilter and MatchesFilePatternDecideRule instead. " +
79              "All URLs that end with the specified pattern(s) will be added " +
80              "to the scope's focus. Default file patterns are:\n.avi, .bmp, " +
81              ".doc, .gif, .jp(e)g, .mid, .mov, .mp2, .mp3, .mp4, .mpeg, " +
82              ".pdf, .png, .ppt, .ram, .rm,.smil, .swf, .tif(f), .wav, .wmv\n" +
83              "It is also possible to specifiy custom regular expressions " +
84              "for this filter, turning it into (effectively) a generic " +
85              "regular expression filter.");
86  
87          String  [] options = new String  [] {ALL, IMAGES, AUDIO, VIDEO, MISC,
88                                              CUSTOM};
89  
90          addElementToDefinition(
91              new SimpleType(ATTR_USE_DEFAULT, "URLs that match selected file " +
92                  "patterns will be crawled. Default file patterns are:\n" +
93                  "Images: .bmp, .gif, .jp(e)g, .png, .tif(f)\nAudio: .mid, " +
94                  ".mp2, .mp3, .mp4, .wav\nVideo: .avi, .mov, .mpeg, .ram, " +
95                  ".rm, .smil, .wmv\nMiscellaneous: .doc, .pdf, .ppt, .swf\n" +
96                  "All: All above patterns\nChoose 'Custom' to specify your own" +
97                  " pattern. These default patterns are case insensitive.",
98                  "All", options));
99  
100         addElementToDefinition(
101             new SimpleType(ATTR_REGEXP, "Custom java regular expression.+n " +
102                     "This regular expression will be used instead of the " +
103                     "supplied pattern groups for matching.\nAn example " +
104                     "of such a regular expression (Miscellaneous):\n" +
105                     ".*(?i)(\\.(doc|pdf|ppt|swf))$\n" +
106                     "Any arbitrary reg.expr. is valid though and will be " +
107                     "applied to the URI.", ""));
108 
109 
110     }
111 
112     /**
113      * @see org.archive.crawler.filter.URIRegExpFilter#getRegexp(java.lang.Object)
114      */
115     protected String   getRegexp(Object   o) {
116         try {
117             String   patternType = (String  )getAttribute(o, ATTR_USE_DEFAULT);
118 
119             if (patternType.equals(ALL)) {
120                 return ALL_DEFAULT_PATTERNS;
121             } else if (patternType.equals(IMAGES)) {
122                 return IMAGES_PATTERNS;
123             }else if (patternType.equals(AUDIO)) {
124                 return AUDIO_PATTERNS;
125             }else if(patternType.equals(VIDEO)) {
126                 return VIDEO_PATTERNS;
127             }else if(patternType.equals(MISC)) {
128                 return MISC_PATTERNS;
129             }else if(patternType.equals(CUSTOM)) {
130                 return (String  ) getAttribute(o, ATTR_REGEXP);
131             }else {
132                 assert false : "Unrecognized pattern type " + patternType +
133                                ". Should never happened!";
134             }
135 
136         } catch (AttributeNotFoundException   e) {
137             logger.log(Level.SEVERE,"necessary setting missing",e);
138         }
139         // Basically the filter is inactive if this occurs (The caller
140         // returns 'false' when regexp is null).
141         return null;  
142     }
143 
144     /**
145      * @see org.archive.crawler.framework.Filter#accepts(java.lang.Object)
146      */
147     public boolean accepts(Object   o) {
148         CrawlURI curi = (o instanceof CrawlURI) ? (CrawlURI) o : null;
149 
150         // Skip the evaluation if the filter is disabled.
151         // Since this filter is primarily used with seed and focus filters
152         // it has to return false when disabled -- unlike Filter's accepts
153         // method.
154         try {
155             if (!((Boolean  ) getAttribute(ATTR_ENABLED, curi)).booleanValue()) {
156                 return false;
157             }
158         } catch (AttributeNotFoundException   e) {
159             logger.severe(e.getMessage());
160         }
161 
162         boolean accept = returnTrueIfMatches(curi) == innerAccepts(o);
163 
164         if (accept && logger.isLoggable(Level.FINEST)) {
165             // Log if filter returns true
166             ComplexType p = this.getParent();
167             if (p instanceof MapType) {
168                 p = p.getParent();
169             }
170             String   msg = this.toString() + " belonging to " + p.toString()
171                          + " accepted " + o.toString();
172             logger.finest(msg);
173         }
174 
175         return accept;
176     }
177 
178 }
179
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags