KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > com > openedit > store > ProductSearch


1 /*
2  * Created on May 2, 2004
3  */
4 package com.openedit.store;
5
6 import java.io.File JavaDoc;
7 import java.io.FileReader JavaDoc;
8 import java.io.IOException JavaDoc;
9 import java.io.StringWriter JavaDoc;
10 import java.text.SimpleDateFormat JavaDoc;
11 import java.util.ArrayList JavaDoc;
12 import java.util.Date JavaDoc;
13 import java.util.HashSet JavaDoc;
14 import java.util.Iterator JavaDoc;
15 import java.util.List JavaDoc;
16 import java.util.Set JavaDoc;
17
18 import org.apache.commons.logging.Log;
19 import org.apache.commons.logging.LogFactory;
20 import org.apache.lucene.analysis.Analyzer;
21 import org.apache.lucene.document.DateTools;
22 import org.apache.lucene.document.Document;
23 import org.apache.lucene.document.Field;
24 import org.apache.lucene.document.DateTools.Resolution;
25 import org.apache.lucene.index.IndexReader;
26 import org.apache.lucene.index.IndexWriter;
27 import org.apache.lucene.index.Term;
28 import org.apache.lucene.queryParser.QueryParser;
29 import org.apache.lucene.search.Hits;
30 import org.apache.lucene.search.Query;
31 import org.apache.lucene.search.Sort;
32 import org.apache.lucene.search.SortComparator;
33 import org.apache.lucene.search.SortField;
34 import org.openedit.money.Money;
35
36 import com.openedit.modules.search.BaseLuceneSearch;
37 import com.openedit.modules.search.CompositeAnalyzer;
38 import com.openedit.modules.search.LuceneHitTracker;
39 import com.openedit.modules.search.RecordLookUpAnalyzer;
40 import com.openedit.store.products.Detail;
41 import com.openedit.store.products.RandomSortComparator;
42 import com.openedit.util.FileUtils;
43 import com.openedit.util.OutputFiller;
44
45 /**
46  * This keeps two indexes. When one is reindexing people can search on the backup one
47  * The main index is A and is used for all edits and deletes
48  * @author cburkey
49  *
50  */
51 public class ProductSearch extends BaseLuceneSearch implements StoreSearcher
52 {
53     private static final Log log = LogFactory.getLog(ProductSearch.class);
54     protected Store fieldStore;
55     protected SimpleDateFormat JavaDoc fieldLuceneDateFormat;
56     protected SortComparator fieldRandomComparator;
57     protected boolean fieldAddProductsToParentCatalog = true;
58     protected Analyzer fieldExactAnalyzer;
59
60     public File JavaDoc getSearchDirectory()
61     {
62         if( fieldSearchDirectory == null)
63         {
64             fieldSearchDirectory = (new File JavaDoc(getStore().getStoreDirectory(), "products/"));
65         }
66         return fieldSearchDirectory;
67     }
68     
69     public LuceneHitTracker search(String JavaDoc inQuery, String JavaDoc inOrdering) throws StoreException
70     {
71         try
72         {
73             //Lucene has problems with searching for dashes or underscores
74 QueryParser parser = new QueryParser("description", getAnalyzer());
75             parser.setDefaultOperator(QueryParser.AND_OPERATOR);
76             Query query1 = parser.parse(inQuery);
77
78             log.info("Searching for: " + query1 );
79
80             Hits hits = null;
81             long start = System.currentTimeMillis();
82             if (inOrdering != null)
83             {
84                 Sort sort = null;
85                 if (inOrdering.equals("priceOrdering"))
86                 {
87                     sort = new Sort(inOrdering, true); //This is a temp hack. We need to add a sort direction indicator
88 }
89                 else if (inOrdering.equals("random"))
90                 {
91                     sort = new Sort();
92                     //SortComparator custom = SampleComparable.getComparator();
93 SortComparator custom = getRandomComparator();
94                     sort.setSort(new SortField("id", custom));
95                 }
96                 else
97                 {
98                     boolean direction = false;
99                     if (inOrdering.endsWith("Down"))
100                     {
101                         direction = true;
102                         inOrdering = inOrdering.substring(0, inOrdering.length() - 4);
103                     }
104                     else if (inOrdering.endsWith("Up"))
105                     {
106                         direction = false;
107                         inOrdering = inOrdering.substring(0, inOrdering.length() - 2);
108                     }
109                     sort = new Sort(inOrdering, direction);
110                 }
111                 hits = getLiveSearcher().search(query1, sort);
112             }
113             else
114             {
115                 hits = getLiveSearcher().search(query1);
116             }
117             long end = System.currentTimeMillis() - start;
118             log.info(hits.length() + " total hits in " + (double) end / 1000D + " seconds");
119
120             LuceneHitTracker tracker = new LuceneHitTracker(hits);
121 // tracker.setQuery(inQuery);
122 // tracker.setOrdering(inOrdering);
123 tracker.setIndexId(getIndexId());
124
125             return tracker;
126         }
127         catch (Exception JavaDoc ex)
128         {
129             log.error(ex);
130             throw new StoreException(ex);
131         }
132     }
133
134     public void updateIndex(Product inProduct) throws StoreException
135     {
136         List JavaDoc all = new ArrayList JavaDoc(1);
137         all.add(inProduct);
138         updateIndex(all);
139     }
140
141     public void updateIndex(List JavaDoc inProducts) throws StoreException
142     {
143         log.info("update index");
144         File JavaDoc indexDir = buildIndexDir("A");
145
146         if (!IndexReader.indexExists(indexDir))
147         {
148             reIndexAll();
149             return;
150         }
151         try
152         {
153             IndexReader reader = IndexReader.open(indexDir); //TODO: Keep it open?
154 try
155             {
156                 for (Iterator JavaDoc iter = inProducts.iterator(); iter.hasNext();)
157                 {
158                     Product product = (Product) iter.next();
159                     Term term = new Term("id", product.getId());
160                     reader.deleteDocuments(term);
161                 }
162             }
163             finally
164             {
165                 reader.close();
166                 IndexReader.unlock(reader.directory());
167             }
168
169             IndexWriter writer = new IndexWriter(indexDir, getAnalyzer(), false);
170             try
171             {
172                 for (Iterator JavaDoc iter = inProducts.iterator(); iter.hasNext();)
173                 {
174                     Product product = (Product) iter.next();
175                     populateProduct(writer, product);
176                 }
177             }
178             finally
179             {
180                 writer.close();
181             }
182             clearIndex();
183
184         }
185         catch (IOException JavaDoc ex)
186         {
187             throw new StoreException(ex);
188         }
189     }
190
191     public void reIndexAll() throws StoreException
192     {
193         try
194         {
195
196             //http://today.java.net/pub/a/today/2003/07/30/LuceneIntro.html?page=last&x-maxdepth=0
197 File JavaDoc existing = buildIndexDir("A");
198             if( IndexReader.indexExists(existing) )
199             {
200                 getLiveSearcher();
201             }
202             log.info("Listing all products");
203
204             List JavaDoc ids = getProductArchive().listAllProductIds();
205             File JavaDoc newIndex = buildIndexDir("new");
206             indexAll(newIndex, ids);
207             FileUtils utils = new FileUtils();
208             utils.deleteAll(existing);
209             utils.move(newIndex, existing);
210             clearIndex();
211
212         }
213         catch (Exception JavaDoc ex)
214         {
215             throw new StoreException(ex);
216         }
217     }
218
219     protected void indexAll(File JavaDoc deadIndexDir, List JavaDoc ids) throws IOException JavaDoc,
220         StoreException
221     {
222         log.info("Reindex started on with " + ids.size() + " products");
223
224         new FileUtils().deleteAll(deadIndexDir);
225         deadIndexDir.mkdirs();
226         IndexWriter writer = new IndexWriter(deadIndexDir, getAnalyzer(), true);
227         //http://www.onjava.com/pub/a/onjava/2003/03/05/lucene.html
228 //http://www.onjava.com/pub/a/onjava/2003/03/05/lucene.html?page=2
229 //writer.mergeFactor = 10;
230 writer.setMergeFactor(100);
231         try
232         {
233             for (Iterator JavaDoc iter = ids.iterator(); iter.hasNext();)
234             {
235                 String JavaDoc id = (String JavaDoc) iter.next();
236                 Product product = getProductArchive().getProduct(id);
237                 if (product != null && product.isAvailable())
238                 {
239                     populateProduct(writer, product);
240                     getProductArchive().clearProduct(product); //remove it from mem
241 }
242                 else
243                 {
244                     log.info("Error loading product:" + id);
245                 }
246             }
247             writer.optimize();
248
249         }
250         finally
251         {
252             writer.close();
253         }
254         //HitCollector
255 log.info("Reindex done");
256     }
257
258     protected void populateProduct(IndexWriter writer, Product product) throws StoreException
259     {
260         if (product.isAvailable())
261         {
262             Document doc = new Document();
263             doc.add(new Field("id", product.getId(), Field.Store.YES, Field.Index.TOKENIZED));
264
265             Money price = product.getYourPrice();
266             if (price != null)
267             {
268                 doc.add(new Field("yourprice", price.toString(), Field.Store.YES,
269                     Field.Index.NO_NORMS));
270                 String JavaDoc shortprice = price.toShortString();
271                 shortprice = shortprice.replaceAll("\\.", "");
272
273                 doc.add(new Field("priceOrdering", pad(shortprice), Field.Store.NO,
274                     Field.Index.NO_NORMS));
275
276                 if (product.isOnSale())
277                 {
278                     Money regular = product.getRetailPrice();
279                     if (regular != null)
280                     {
281                         doc.add(new Field("regularprice", regular.toString(), Field.Store.YES,
282                             Field.Index.NO_NORMS));
283                     }
284                 }
285             }
286             populateCatalog(doc, product);
287
288             //this may be invalid field of -1 but we still need to add it for the search to work
289 if (product.getOrdering() != -1)
290             {
291                 doc.add(new Field("ordering", Integer.toString(product.getOrdering()),
292                     Field.Store.NO, Field.Index.NO_NORMS));
293             }
294
295             populateDescription(doc, product);
296             populateProperties(doc, product);
297             /* StringBuffer sizes = new StringBuffer();
298              for (Iterator iters = product.getSizes().iterator(); iters.hasNext();) {
299              String size = (String) iters.next();
300              sizes.append(size);
301              sizes.append(" ");
302              }
303              doc.add(Field.Text("sizes", sizes.toString().trim() ) );
304              */
305             StringBuffer JavaDoc items = new StringBuffer JavaDoc();
306             for (Iterator JavaDoc iter = product.getInventoryItems().iterator(); iter.hasNext();)
307             {
308                 InventoryItem item = (InventoryItem) iter.next();
309                 items.append(item.getSku());
310                 items.append(" ");
311             }
312             if (items.length() > 0)
313             {
314                 doc.add(new Field("items", items.toString().trim(), Field.Store.NO,
315                     Field.Index.TOKENIZED));
316             }
317             try
318             {
319                 writer.addDocument(doc);
320                 log.debug("Indexed " + product.getId());
321             }
322             catch (IOException JavaDoc ex)
323             {
324                 throw new StoreException(ex);
325             }
326         }
327     }
328
329     protected String JavaDoc pad(String JavaDoc inShortprice)
330     {
331         String JavaDoc all = "0000000000000" + inShortprice;
332         String JavaDoc cut = all.substring(all.length() - 10); //10 is the max width of integers
333 return cut;
334     }
335
336     protected void populateProperties(Document inDoc, Product inProduct) throws StoreException
337     {
338         List JavaDoc list = getProductArchive().getPropertyDetails().findIndexProperties();
339         for (Iterator JavaDoc iter = list.iterator(); iter.hasNext();)
340         {
341             Detail det = (Detail) iter.next();
342             if (det.isIndex())
343             {
344                 String JavaDoc prop = inProduct.get(det.getId());
345                 if (prop != null)
346                 {
347                     if (det.isDate())
348                     {
349                         String JavaDoc date = inProduct.get(det.getId());
350                         if (date != null && date.length() > 0)
351                         {
352                             try
353                             {
354                                 Date JavaDoc realdate = det.getDateFormat().parse(date);
355                                 prop = DateTools.dateToString(realdate, Resolution.SECOND);
356                                 inDoc.add(new Field(det.getId(), prop, Field.Store.YES,
357                                     Field.Index.NO_NORMS));
358                             }
359                             catch (Exception JavaDoc ex)
360                             {
361                                 throw new StoreException(ex);
362                             }
363                         }
364                     }
365                     else if( det.isStored() )
366                     {
367                         inDoc.add(new Field(det.getId(), prop, Field.Store.YES,
368                             Field.Index.TOKENIZED));
369                     }
370                     else
371                     {
372                         inDoc.add(new Field(det.getId(), prop, Field.Store.NO,Field.Index.TOKENIZED));
373                     }
374                 }
375             }
376         }
377     }
378
379     protected void populateCatalog(Document doc, Product item)
380     {
381         StringBuffer JavaDoc buffer = new StringBuffer JavaDoc();
382         Set JavaDoc catalogs = buildCatalogSet(item);
383         for (Iterator JavaDoc iter = catalogs.iterator(); iter.hasNext();)
384         {
385             Category catalog = (Category) iter.next();
386             buffer.append(catalog.getId());
387             buffer.append(" ");
388         }
389         //Add in all the catalogs, price, gender, image on disk?, name+ full text
390 if (buffer.length() > 0)
391         {
392             doc
393                 .add(new Field("category", buffer.toString(), Field.Store.NO, Field.Index.TOKENIZED));
394         }
395         /* Not used any more
396          if ( item.getDepartment() != null)
397          {
398          doc.add( new Field("department", item.getDepartment(), Field.Store.YES, Field.Index.TOKENIZED));
399          }
400          */
401     }
402
403     protected Set JavaDoc buildCatalogSet(Product inProduct)
404     {
405         HashSet JavaDoc allCatalogs = new HashSet JavaDoc();
406         Set JavaDoc catalogs = inProduct.getCatalogs();
407         allCatalogs.addAll(catalogs);
408         for (Iterator JavaDoc iter = catalogs.iterator(); iter.hasNext();)
409         {
410             Category catalog = (Category) iter.next();
411             buildCatalogSet(catalog, allCatalogs);
412         }
413         return allCatalogs;
414     }
415
416     protected void buildCatalogSet(Category inCatalog, Set JavaDoc inCatalogSet)
417     {
418         inCatalogSet.add(inCatalog);
419         Category parent = inCatalog.getParentCatalog();
420         if (parent != null && !parent.getId().equals("index") && isAddProductsToParentCatalog())
421         {
422             buildCatalogSet(parent, inCatalogSet);
423         }
424     }
425
426     protected void populateDescription(Document doc, Product product) throws StoreException
427     {
428         if (product.getName() != null)
429         {
430             doc.add(new Field("name", product.getName(), Field.Store.YES, Field.Index.NO_NORMS));
431         }
432         String JavaDoc htmlPath = getProductArchive().getProductPathFinder().idToPath(product.getId())
433             + ".html";
434         //Low level reading in of text
435 StringBuffer JavaDoc fullDesc = new StringBuffer JavaDoc();
436         fullDesc.append(product.getName());
437         populateKeywords(fullDesc, product);
438         //add a bunch of stuff to the full text field
439 File JavaDoc descriptionFile = new File JavaDoc(getSearchDirectory(), htmlPath);
440         if (descriptionFile.exists() || descriptionFile.length() > 0)
441         {
442             FileReader JavaDoc descread = null;
443             try
444             {
445                 descread = new FileReader JavaDoc(descriptionFile);
446                 StringWriter JavaDoc out = new StringWriter JavaDoc();
447                 new OutputFiller().fill(descread, out);
448                 fullDesc.append(out.toString());
449             }
450             catch (Exception JavaDoc ex)
451             {
452                 throw new StoreException(ex);
453             }
454             finally
455             {
456                 FileUtils.safeClose(descread);
457             }
458         }
459         fullDesc.append(' ');
460         for (Iterator JavaDoc iter = product.getCatalogs().iterator(); iter.hasNext();)
461         {
462             Category cat = (Category) iter.next();
463             fullDesc.append(cat.getName());
464             fullDesc.append(' ');
465         }
466         doc
467             .add(new Field("description", fullDesc.toString(), Field.Store.NO,
468                 Field.Index.TOKENIZED));
469     }
470
471     protected void populateKeywords(StringBuffer JavaDoc inFullDesc, Product inProduct)
472         throws StoreException
473     {
474         inFullDesc.append(' ');
475         if (inProduct.getKeywords() != null)
476         {
477             String JavaDoc desc = inProduct.getKeywords();
478             desc = desc.replace('/', ' ');
479             desc = desc.replace('\\', ' ');
480             inFullDesc.append(desc);
481             inFullDesc.append(' ');
482         }
483
484         List JavaDoc list = getProductArchive().getPropertyDetails().getDetails();
485         for (Iterator JavaDoc iter = list.iterator(); iter.hasNext();)
486         {
487             Detail det = (Detail) iter.next();
488             if (det.isKeyword())
489             {
490                 String JavaDoc prop = inProduct.get(det.getId());
491                 if (prop != null)
492                 {
493                     inFullDesc.append(prop);
494                     inFullDesc.append(' ');
495                 }
496             }
497         }
498
499     }
500
501     public ProductArchive getProductArchive()
502     {
503         return fieldStore.getProductArchive();
504     }
505
506     public Store getStore()
507     {
508         return fieldStore;
509     }
510
511     public void setStore(Store inStore)
512     {
513         fieldStore = inStore;
514     }
515
516     public void deleteFromIndex(Product inProduct) throws StoreException
517     {
518         log.info("delete from index");
519         File JavaDoc indexDir = buildIndexDir("A");
520
521         try
522         {
523             IndexReader reader = IndexReader.open(indexDir);
524             try
525             {
526                 Term term = new Term("id", inProduct.getId());
527                 reader.deleteDocuments(term);
528             }
529             finally
530             {
531                 reader.close();
532                 IndexReader.unlock(reader.directory());
533             }
534             clearIndex();
535         }
536         catch (IOException JavaDoc ex)
537         {
538             throw new StoreException(ex);
539         }
540     }
541
542     //this is actually a date time
543 public SimpleDateFormat JavaDoc getLuceneDateFormat()
544     {
545         if (fieldLuceneDateFormat == null)
546         {
547             fieldLuceneDateFormat = new SimpleDateFormat JavaDoc("yyyyMMddHHmmss");
548         }
549         return fieldLuceneDateFormat;
550     }
551
552     public SortComparator getRandomComparator()
553     {
554         if (fieldRandomComparator == null)
555         {
556             fieldRandomComparator = new RandomSortComparator();
557         }
558         return fieldRandomComparator;
559     }
560
561     public void setRandomComparator(SortComparator inRandomComparator)
562     {
563         fieldRandomComparator = inRandomComparator;
564     }
565
566     public boolean isAddProductsToParentCatalog()
567     {
568         return fieldAddProductsToParentCatalog;
569     }
570
571     public void setAddProductsToParentCatalog(boolean inAddProductsToParentCatalog)
572     {
573         fieldAddProductsToParentCatalog = inAddProductsToParentCatalog;
574     }
575
576     public Analyzer getAnalyzer()
577     {
578         if (fieldAnalyzer == null)
579         {
580             CompositeAnalyzer composite = new CompositeAnalyzer();
581             //composite.setAnalyzer("description", getGenericAnalyzer() );
582 composite.setAnalyzer("category", getExactAnalyzer());
583             composite.setAnalyzer("items", getExactAnalyzer());
584             composite.setAnalyzer("priceOrdering", getExactAnalyzer());
585             composite.setAnalyzer("yourprice", getExactAnalyzer());
586             composite.setAnalyzer("department", getExactAnalyzer());
587             composite.setAnalyzer("date", getExactAnalyzer());
588             composite.setAnalyzer("id", getExactAnalyzer());
589             composite.setAnalyzer("cumulusid", getExactAnalyzer());
590             fieldAnalyzer = composite;
591         }
592         return fieldAnalyzer;
593     }
594
595     protected Analyzer getExactAnalyzer()
596     {
597         if (fieldExactAnalyzer == null)
598         {
599             /**
600              * This is case insesitive.
601              * Only matches words within a space.
602              * does not remove dots or weird chars
603              */
604             fieldExactAnalyzer = new RecordLookUpAnalyzer();
605         }
606         return fieldExactAnalyzer;
607     }
608 }
Popular Tags