KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > apache > lucene > analysis > StopFilter


1 package org.apache.lucene.analysis;
2
3 /**
4  * Copyright 2004 The Apache Software Foundation
5  *
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  */
18
19 import java.io.IOException JavaDoc;
20 import java.util.HashSet JavaDoc;
21 import java.util.Hashtable JavaDoc;
22 import java.util.Set JavaDoc;
23
24 /**
25  * Removes stop words from a token stream.
26  */
27
28 public final class StopFilter extends TokenFilter {
29
30   private final Set JavaDoc stopWords;
31   private final boolean ignoreCase;
32
33     /**
34      * Construct a token stream filtering the given input.
35      */
36     public StopFilter(TokenStream input, String JavaDoc [] stopWords)
37     {
38         this(input, stopWords, false);
39     }
40
41   /**
42    * Constructs a filter which removes words from the input
43    * TokenStream that are named in the array of words.
44    */
45   public StopFilter(TokenStream in, String JavaDoc[] stopWords, boolean ignoreCase) {
46     super(in);
47     this.ignoreCase = ignoreCase;
48     this.stopWords = makeStopSet(stopWords, ignoreCase);
49   }
50
51   /**
52    * Constructs a filter which removes words from the input
53    * TokenStream that are named in the Hashtable.
54    *
55    * @deprecated Use {@link #StopFilter(TokenStream, Set)} instead
56    */
57   public StopFilter(TokenStream in, Hashtable JavaDoc stopTable) {
58     this(in, stopTable, false);
59   }
60   /**
61    * Constructs a filter which removes words from the input
62    * TokenStream that are named in the Hashtable.
63    * If ignoreCase is true, all keys in the stopTable should already
64    * be lowercased.
65    * @deprecated Use {@link #StopFilter(TokenStream, Set)} instead
66    */
67   public StopFilter(TokenStream in, Hashtable JavaDoc stopTable, boolean ignoreCase) {
68     this(in, stopTable.keySet(), ignoreCase);
69   }
70
71     /**
72      * Construct a token stream filtering the given input.
73      * @param input
74      * @param stopWords The set of Stop Words, as Strings. If ignoreCase is true, all strings should be lower cased
75      * @param ignoreCase -Ignore case when stopping. The stopWords set must be setup to contain only lower case words
76      */
77     public StopFilter(TokenStream input, Set JavaDoc stopWords, boolean ignoreCase)
78     {
79         super(input);
80         this.ignoreCase = ignoreCase;
81         this.stopWords = stopWords;
82     }
83
84   /**
85    * Constructs a filter which removes words from the input
86    * TokenStream that are named in the Set.
87    * It is crucial that an efficient Set implementation is used
88    * for maximum performance.
89    *
90    * @see #makeStopSet(java.lang.String[])
91    */
92   public StopFilter(TokenStream in, Set JavaDoc stopWords) {
93     this(in, stopWords, false);
94   }
95   /**
96    * Builds a Hashtable from an array of stop words,
97    * appropriate for passing into the StopFilter constructor.
98    * This permits this table construction to be cached once when
99    * an Analyzer is constructed.
100    *
101    * @deprecated Use {@link #makeStopSet(String[])} instead.
102    */
103   public static final Hashtable JavaDoc makeStopTable(String JavaDoc[] stopWords) {
104       return makeStopTable(stopWords, false);
105   }
106     
107     /**
108     * Builds a Hashtable from an array of stop words,
109     * appropriate for passing into the StopFilter constructor.
110     * This permits this table construction to be cached once when
111     * an Analyzer is constructed.
112   * @deprecated Use {@link #makeStopSet(java.lang.String[], boolean)} instead.
113  */
114   public static final Hashtable JavaDoc makeStopTable(String JavaDoc [] stopWords, boolean ignoreCase) {
115       Hashtable JavaDoc stopTable = new Hashtable JavaDoc(stopWords.length);
116       for (int i = 0; i < stopWords.length; i++)
117       {
118           String JavaDoc stopWord = ignoreCase ? stopWords[i].toLowerCase() : stopWords[i];
119           stopTable.put(stopWord, stopWord);
120       }
121     return stopTable;
122   }
123
124   /**
125    * Builds a Set from an array of stop words,
126    * appropriate for passing into the StopFilter constructor.
127    * This permits this stopWords construction to be cached once when
128    * an Analyzer is constructed.
129    *
130    * @see #makeStopSet(java.lang.String[], boolean) passing false to ignoreCase
131    */
132   public static final Set JavaDoc makeStopSet(String JavaDoc[] stopWords) {
133     return makeStopSet(stopWords, false);
134   }
135     
136   /**
137    *
138     * @param stopWords
139    * @param ignoreCase If true, all words are lower cased first.
140    * @return a Set containing the words
141    */
142   public static final Set JavaDoc makeStopSet(String JavaDoc[] stopWords, boolean ignoreCase) {
143     HashSet JavaDoc stopTable = new HashSet JavaDoc(stopWords.length);
144     for (int i = 0; i < stopWords.length; i++)
145       stopTable.add(ignoreCase ? stopWords[i].toLowerCase() : stopWords[i]);
146     return stopTable;
147   }
148
149   /**
150    * Returns the next input Token whose termText() is not a stop word.
151    */
152   public final Token next() throws IOException JavaDoc {
153     // return the first non-stop word found
154 for (Token token = input.next(); token != null; token = input.next())
155     {
156         String JavaDoc termText = ignoreCase ? token.termText.toLowerCase() : token.termText;
157         if (!stopWords.contains(termText))
158           return token;
159     }
160     // reached EOS -- return null
161 return null;
162   }
163 }
164
Popular Tags