KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > com > mvnforum > search > post > PostIndexer


1 /*
2  * $Header: /cvsroot/mvnforum/mvnforum/src/com/mvnforum/search/post/PostIndexer.java,v 1.12 2006/04/14 17:05:27 minhnn Exp $
3  * $Author: minhnn $
4  * $Revision: 1.12 $
5  * $Date: 2006/04/14 17:05:27 $
6  *
7  * ====================================================================
8  *
9  * Copyright (C) 2002-2006 by MyVietnam.net
10  *
11  * All copyright notices regarding mvnForum MUST remain
12  * intact in the scripts and in the outputted HTML.
13  * The "powered by" text/logo with a link back to
14  * http://www.mvnForum.com and http://www.MyVietnam.net in
15  * the footer of the pages MUST remain visible when the pages
16  * are viewed on the internet or intranet.
17  *
18  * This program is free software; you can redistribute it and/or modify
19  * it under the terms of the GNU General Public License as published by
20  * the Free Software Foundation; either version 2 of the License, or
21  * any later version.
22  *
23  * This program is distributed in the hope that it will be useful,
24  * but WITHOUT ANY WARRANTY; without even the implied warranty of
25  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26  * GNU General Public License for more details.
27  *
28  * You should have received a copy of the GNU General Public License
29  * along with this program; if not, write to the Free Software
30  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
31  *
32  * Support can be obtained from support forums at:
33  * http://www.mvnForum.com/mvnforum/index
34  *
35  * Correspondence and Marketing Questions can be sent to:
36  * info at MyVietnam net
37  *
38  * @author: Minh Nguyen
39  * @author: Dejan Krsmanovic dejan_krsmanovic@yahoo.com
40  */
41 package com.mvnforum.search.post;
42
43 import java.io.IOException JavaDoc;
44
45 import net.myvietnam.mvncore.exception.SearchException;
46 import net.myvietnam.mvncore.util.DateUtil;
47 import net.myvietnam.mvncore.util.TimerUtil;
48
49 import org.apache.commons.logging.Log;
50 import org.apache.commons.logging.LogFactory;
51 import org.apache.lucene.analysis.Analyzer;
52 import org.apache.lucene.analysis.standard.StandardAnalyzer;
53 import org.apache.lucene.document.*;
54 import org.apache.lucene.index.*;
55 import org.apache.lucene.search.IndexSearcher;
56 import org.apache.lucene.store.Directory;
57
58 import com.mvnforum.*;
59 import com.mvnforum.db.PostBean;
60 import com.mvnforum.search.IntegerFilter;
61
62 public class PostIndexer
63 {
64     private static Log log = LogFactory.getLog(PostIndexer.class);
65
66     //Field names (used for indexing)
67 public static final String JavaDoc FIELD_POST_ID = "postID";
68     public static final String JavaDoc FIELD_THREAD_ID = "threadID";
69     public static final String JavaDoc FIELD_FORUM_ID = "forumID";
70     public static final String JavaDoc FIELD_MEMBER_ID = "memberID";
71     public static final String JavaDoc FIELD_POST_TOPIC = "postTopic";
72     public static final String JavaDoc FIELD_POST_BODY = "postBody";
73     public static final String JavaDoc FIELD_POST_DATE = "postDate";
74
75     public static final String JavaDoc FIELD_WITH_ATTACHMENT = "withAttachment";
76
77     public static final String JavaDoc FIELD_ATTACHMENT_COUNT = "attachmentCount";
78
79     //public static final String PROPERTY_SEARCH_PATH = "search.path";
80 //public static final String PROPERTY_SEARCH_AUTOINDEX = "search.autoindex";
81
82     //Timer is used for scheduling jobs
83 private static Analyzer analyzer;
84
85     private static long lastOptimizeTime = 0;
86
87     static {
88         initializeAnalyzer();
89     }
90
91     public static void scheduleAddPostTask(PostBean postBean) {
92         AddUpdatePostIndexTask task = new AddUpdatePostIndexTask(postBean, AddUpdatePostIndexTask.OPERATION_ADD);
93         TimerUtil.getInstance().schedule(task, 0);
94     }
95
96     public static void scheduleUpdatePostTask(PostBean postBean) {
97         AddUpdatePostIndexTask task = new AddUpdatePostIndexTask(postBean, AddUpdatePostIndexTask.OPERATION_UPDATE);
98         TimerUtil.getInstance().schedule(task, 0);
99     }
100
101     public static void scheduleDeletePostTask(int objectID, int objectType) {
102         DeletePostIndexTask task = new DeletePostIndexTask(objectID, objectType);
103         TimerUtil.getInstance().schedule(task, 0);
104     }
105
106     public static void scheduleUpdateThreadTask(int threadID) {
107         UpdateThreadTask task = new UpdateThreadTask(threadID);
108         TimerUtil.getInstance().schedule(task, 0);
109     }
110
111     public static void scheduleRebuildIndexTask() {
112         int maxPostID = 0;
113         RebuildPostIndexTask task = new RebuildPostIndexTask(maxPostID);
114         TimerUtil.getInstance().schedule(task, 0);
115     }
116
117     static Analyzer getAnalyzer() {
118         return analyzer;
119     }
120
121     /**
122      * This class will load analyzer when starting. If specified analyzer class
123      * cannot be loaded then default analyzer will be used.
124      */
125     private static void initializeAnalyzer() {
126         String JavaDoc analyzerClassName = MVNForumFactoryConfig.getLuceneAnalyzerClassName();
127         if ( (analyzerClassName == null) || (analyzerClassName.equals("")) ) {
128             //create standard analyzer
129 //String[] stopWords = this.loadStopWords();
130 analyzer = new StandardAnalyzer();
131             log.debug("Using StandardAnalyzer for indexing");
132         } else {
133             //try to create specified analyzer
134 try {
135                 log.debug("About to load Analyzer [" + analyzerClassName + "] for indexing");
136                 analyzer = (Analyzer) Class.forName(analyzerClassName).newInstance();
137             } catch (Exception JavaDoc e) {
138                 log.warn("Cannot load " + analyzerClassName + ". Loading StandardAnalyzer");
139                 analyzer = new StandardAnalyzer();
140             }
141         }
142     }
143
144     /**
145      * This method is used for getting new IndexWriter. It can create new index
146      * or add post to existing index. Creating new index will delete previous so it
147      * should be used for rebuilding index.
148      * @param create - true if new index should be created.
149      * - false for adding posts to existing index
150      * @return IndexWriter object that is used for adding posts to index
151      */
152     static IndexWriter getIndexWriter(Directory directory, boolean create) throws SearchException {
153         
154         IndexWriter writer = null;
155
156         //If create = false, we will create IndexWriter with false argument
157 if (create == false) {
158             try {
159                 writer = new IndexWriter(directory, analyzer, false);
160                 if (MVNForumConfig.getSearchPostIndexType() == MVNForumGlobal.SEARCH_INDEX_TYPE_DISK) {
161                     writer.setUseCompoundFile(true);
162                 }
163                 return writer;
164             } catch (IOException JavaDoc e) {
165                 log.warn("Cannot open existed index. New index will be created.", e);
166                 //Ignore Exception. We will try to create index with true parameter
167 }
168         }
169         // We are here in two cases: We wanted to create new index or because
170 // index doesn't existed
171 try {
172             //This will create new index and delete existing
173 //writer = new IndexWriter(MVNForumConfig.getSearchPostIndexDir(true), analyzer, true);
174 writer = new IndexWriter(directory, analyzer, true);// actually the directory should be 'create' = true
175 if (MVNForumConfig.getSearchPostIndexType() == MVNForumGlobal.SEARCH_INDEX_TYPE_DISK) {
176                 writer.setUseCompoundFile(true);
177             }
178             return writer;
179         } catch (IOException JavaDoc e) {
180             //@todo : localize me
181 log.error("IOException during get index writer", e);
182             throw new SearchException("Error while creating index writer");
183         }
184     }
185
186     /**
187      * This method is used for adding single post to index
188      * Note: this method doesnt close the writer
189      * @param post A post that should be indexed
190      * @param writer IndexWriter that is used for storing
191      * @throws SearchException
192      */
193     static void doIndexPost(PostBean post, IndexWriter writer) throws SearchException {
194
195         if (post == null) return;
196         //Post must include topic and body. If not then we have nothing to index.
197 if ( (post.getPostTopic() == null || post.getPostTopic().equals("")) ||
198              (post.getPostBody() == null || post.getPostBody().equals(""))) {
199             return;
200         }
201
202         //Each post will be represented as a document
203 Document postDocument = new Document();
204         //Document has following fields that could be queried on
205 postDocument.add(Field.Keyword(FIELD_POST_ID, Integer.toString(post.getPostID())));
206         postDocument.add(Field.Keyword(FIELD_THREAD_ID, Integer.toString(post.getThreadID())));
207         postDocument.add(Field.Keyword(FIELD_FORUM_ID, Integer.toString(post.getForumID())));
208         postDocument.add(Field.Keyword(FIELD_MEMBER_ID, Integer.toString(post.getMemberID())));
209         postDocument.add(Field.Keyword(FIELD_WITH_ATTACHMENT, new Boolean JavaDoc(post.getPostAttachCount()>0).toString()));// make it compilable on JDK 1.3
210 postDocument.add(Field.Keyword(FIELD_ATTACHMENT_COUNT, IntegerFilter.intToString(post.getPostAttachCount())));
211         //postDocument.add(Field.Keyword(FIELD_ATTACHMENT_COUNT, Integer.toString(post.getPostAttachCount())));
212
213         //document body and title is not stored since we can retrieve them from database
214 postDocument.add(Field.UnStored(FIELD_POST_TOPIC, post.getPostTopic()));
215         postDocument.add(Field.UnStored(FIELD_POST_BODY, post.getPostBody()));
216         //add date field
217 postDocument.add(Field.Keyword(FIELD_POST_DATE, DateField.dateToString(post.getPostCreationDate())));
218
219         //now we have created document with fields so we can store it
220 try {
221             writer.addDocument(postDocument);
222         } catch (IOException JavaDoc e) {
223             log.error("PostIndexer.doIndexPost failed", e);
224             //@todo : localize me
225 throw new SearchException("Error writing new post to index");
226         }
227     }
228
229     /**
230      * Add single post to index
231      * @param post
232      * @throws SearchException
233      */
234     static void addPostToIndex(PostBean post) throws SearchException, IOException JavaDoc {
235         
236         Directory directory = null;
237         IndexWriter writer = null;
238         try {
239             directory = MVNForumConfig.getSearchPostIndexDir();
240             writer = getIndexWriter(directory, false);
241             if (writer == null) {
242                 log.warn("Cannot get the IndexWriter");
243                 return;
244             }
245             doIndexPost(post, writer);
246
247             // now check if we should optimize index (each hour)
248 long now = System.currentTimeMillis();
249             long timeFromLastOptimize = now - lastOptimizeTime;
250             if (timeFromLastOptimize > DateUtil.HOUR) {
251                 log.debug("writer.optimize() called in addPostToIndex");
252                 writer.optimize();
253                 lastOptimizeTime = now;
254             }
255         } catch (SearchException ex) {
256             throw ex;
257         } finally {
258             if (writer != null) {
259                 try {
260                     writer.close();
261                 } catch (IOException JavaDoc e) {
262                     log.debug("Error closing Lucene IndexWriter", e);
263                 }
264             }
265             if (directory != null) {
266                 try {
267                     directory.close();
268                 } catch (IOException JavaDoc e) {
269                     log.debug("Cannot close directory.", e);
270                 }
271             }
272         }
273     }
274
275     /**
276      * This method is used for deleting post from index.
277      * @param postID id of the post that should be deleted
278      * @throws SearchException
279      */
280     static void deletePostFromIndex(int postID) throws SearchException {
281
282         Directory directory = null;
283         IndexReader reader = null;
284         try {
285             directory = MVNForumConfig.getSearchPostIndexDir();
286             reader = IndexReader.open(directory);
287             if (reader == null) {
288                 log.warn("Cannot get the IndexReader");
289                 return;
290             }
291
292             Term term = new Term(FIELD_POST_ID, String.valueOf(postID));
293             int deletedCount = reader.delete(term);
294             log.debug("deletePostFromIndex: deleted posts = " + deletedCount);
295         } catch (IOException JavaDoc e) {
296             //@todo : localize me
297 throw new SearchException("Error trying to delete post with postID = " + postID);
298         } finally {
299             if (reader != null) {
300                 try {
301                     reader.close();
302                 } catch (IOException JavaDoc e) {
303                     log.debug("Error closing Lucene IndexReader", e);
304                 }
305             }
306             if (directory != null) {
307                 try {
308                     directory.close();
309                 } catch (IOException JavaDoc e) {
310                     log.debug("Cannot close directory.", e);
311                 }
312             }
313         }
314     }
315
316     /**
317      * This method is used for deleting all posts in a thread from index.
318      * @param threadID id of the thread that should be deleted
319      * @throws SearchException
320      */
321     static void deleteThreadFromIndex(int threadID) throws SearchException {
322         
323         Directory directory = null;
324         IndexReader reader = null;
325         try {
326             directory = MVNForumConfig.getSearchPostIndexDir();
327             reader = IndexReader.open(directory);
328             if (reader == null) {
329                 log.warn("Cannot get the IndexReader");
330                 return;
331             }
332
333             Term term = new Term(FIELD_THREAD_ID, String.valueOf(threadID));
334             int deletedCount = reader.delete(term);
335             log.debug("deleteThreadFromIndex: deleted posts = " + deletedCount);
336         } catch (IOException JavaDoc e) {
337             //@todo : localize me
338 throw new SearchException("Error trying to delete posts in index with threadID = " + threadID);
339         } finally {
340             if (reader != null) {
341                 try {
342                     reader.close();
343                 } catch (IOException JavaDoc e) {
344                     log.debug("Error closing Lucene IndexReader", e);
345                 }
346             }
347             if (directory != null) {
348                 try {
349                     directory.close();
350                 } catch (IOException JavaDoc e) {
351                     log.debug("Cannot close directory.", e);
352                 }
353             }
354         }
355     }
356
357     /**
358      * This method is used for deleting all posts in a forum from index.
359      * @param forumID id of the forum that should be deleted
360      * @throws SearchException
361      */
362     static void deleteForumFromIndex(int forumID) throws SearchException {
363         
364         Directory directory = null;
365         IndexReader reader = null;
366         try {
367             directory = MVNForumConfig.getSearchPostIndexDir();
368             reader = IndexReader.open(directory);
369             if (reader == null) {
370                 log.warn("Cannot get the IndexReader");
371                 return;
372             }
373
374             Term term = new Term(FIELD_FORUM_ID, String.valueOf(forumID));
375             int deletedCount = reader.delete(term);
376             log.debug("deleteForumFromIndex: deleted posts = " + deletedCount);
377         } catch (IOException JavaDoc e) {
378             //@todo : localize me
379 throw new SearchException("Error trying to delete posts in index with forumID = " + forumID);
380         } finally {
381             if (reader != null) {
382                 try {
383                     reader.close();
384                 } catch (IOException JavaDoc e) {
385                     log.debug("Error closing Lucene IndexReader", e);
386                 }
387             }
388             if (directory != null) {
389                 try {
390                     directory.close();
391                 } catch (IOException JavaDoc e) {
392                     log.debug("Cannot close directory.", e);
393                 }
394             }
395         }
396     }
397
398     public static int getNumDocs() {
399         
400         int numDocs = -1;
401         Directory directory = null;
402         IndexReader reader = null;
403         try {
404             directory = MVNForumConfig.getSearchPostIndexDir();
405             reader = IndexReader.open(directory);
406             if (reader == null) {
407                 log.warn("Cannot get the IndexReader");
408                 return -1;
409             }
410             numDocs = reader.numDocs();
411         } catch (IOException JavaDoc ioe) {
412             //ignore
413 } finally {
414             if (reader != null) {
415                 try {
416                     reader.close();
417                 } catch (IOException JavaDoc e) {
418                     log.debug("Error closing Lucene IndexReader", e);
419                 }
420             }
421             if (directory != null) {
422                 try {
423                     directory.close();
424                 } catch (IOException JavaDoc e) {
425                     log.debug("Cannot close directory.", e);
426                 }
427             }
428         }
429         return numDocs;
430     }
431
432 }
433
Free Books   Free Magazines  
Popular Tags