Java > Open Source Codes > org > eclipse > help > internal > search > IndexingOperation


1 /*******************************************************************************
2  * Copyright (c) 2000, 2007 IBM Corporation and others.
3  * All rights reserved. This program and the accompanying materials
4  * are made available under the terms of the Eclipse Public License v1.0
5  * which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/epl-v10.html
7  *
8  * Contributors:
9  * IBM Corporation - initial API and implementation
10  *******************************************************************************/
11 package org.eclipse.help.internal.search;
12
13 import java.net.URL ;
14 import java.util.ArrayList ;
15 import java.util.Collection ;
16 import java.util.Collections ;
17 import java.util.HashMap ;
18 import java.util.HashSet ;
19 import java.util.Iterator ;
20 import java.util.Map ;
21 import java.util.Set ;
22
23 import org.eclipse.core.runtime.IConfigurationElement;
24 import org.eclipse.core.runtime.IExtensionRegistry;
25 import org.eclipse.core.runtime.IProgressMonitor;
26 import org.eclipse.core.runtime.IStatus;
27 import org.eclipse.core.runtime.InvalidRegistryObjectException;
28 import org.eclipse.core.runtime.MultiStatus;
29 import org.eclipse.core.runtime.OperationCanceledException;
30 import org.eclipse.core.runtime.Platform;
31 import org.eclipse.core.runtime.SubProgressMonitor;
32 import org.eclipse.help.ITocContribution;
33 import org.eclipse.help.ITopic;
34 import org.eclipse.help.internal.base.BaseHelpSystem;
35 import org.eclipse.help.internal.base.HelpBasePlugin;
36 import org.eclipse.help.internal.base.HelpBaseResources;
37 import org.eclipse.help.internal.base.util.HelpProperties;
38 import org.eclipse.help.internal.protocols.HelpURLConnection;
39 import org.eclipse.help.internal.toc.Toc;
40 import org.eclipse.help.internal.toc.TocFileProvider;
41 import org.eclipse.help.search.LuceneSearchParticipant;
42
43 /**
44  * Indexing Operation represents a long operation, which performs indexing of
45  * the group (Collection) of documents. It is used Internally by SlowIndex and
46  * returned by its getIndexUpdateOperation() method.
47  */
48 class IndexingOperation {
49     
50     private static final String ELEMENT_NAME_INDEX = "index"; //$NON-NLS-1$
51 private static final String ATTRIBUTE_NAME_PATH = "path"; //$NON-NLS-1$
52
53     private int numAdded;
54     private int numRemoved;
55     private SearchIndex index = null;
56
57     /**
58      * Construct indexing operation.
59      *
60      * @param ix
61      * ISearchIndex already opened
62      */
63     public IndexingOperation(SearchIndex ix) {
64         this.index = ix;
65     }
66
67     private void checkCancelled(IProgressMonitor pm)
68             throws OperationCanceledException {
69         if (pm.isCanceled())
70             throw new OperationCanceledException();
71     }
72
73     /**
74      * Executes indexing, given the progress monitor.
75      *
76      * @param pm
77      * progres monitor to be used during this long operation for
78      * reporting progress
79      * @throws OperationCanceledException
80      * if indexing was cancelled
81      */
82     protected void execute(IProgressMonitor pm)
83             throws OperationCanceledException, IndexingException {
84         checkCancelled(pm);
85         Collection staleDocs = getRemovedDocuments(index);
86         numRemoved = staleDocs.size();
87         Collection newDocs = getAddedDocuments(index);
88         numAdded = newDocs.size();
89
90         // if collection is empty, we may return right away
91 // need to check if we have to do anything to the progress monitor
92 if (numRemoved + numAdded <= 0) {
93             pm.done();
94             BaseHelpSystem.getLocalSearchManager().clearSearchParticipants();
95             return;
96         }
97         pm.beginTask(HelpBaseResources.UpdatingIndex, numRemoved + 10
98                 * numAdded);
99
100         // 1. remove all documents for plugins changed (including change in a
101 // fragment)
102 removeStaleDocuments(new SubProgressMonitor(pm, numRemoved), staleDocs);
103         checkCancelled(pm);
104         // 2. merge prebult plugin indexes and addjust
105 addNewDocuments(new SubProgressMonitor(pm, 10 * numAdded), newDocs,
106                 staleDocs.size() == 0);
107
108         pm.done();
109         BaseHelpSystem.getLocalSearchManager().clearSearchParticipants();
110     }
111
112     private Map calculateNewToRemove(Collection newDocs, Map prebuiltDocs) {
113         // Calculate document that were in prebuilt indexes, but are not in
114 // TOCs. (prebuiltDocs - newDocs)
115 /*
116          * Map. Keys are /pluginid/href of docs to delete. Values are null to
117          * delete completely, or String[] of indexIds with duplicates of the
118          * document
119          */
120         Map docsToDelete = prebuiltDocs;
121         ArrayList prebuiltHrefs = new ArrayList (prebuiltDocs.keySet());
122         for (int i = 0; i < prebuiltHrefs.size(); i++) {
123             String href = (String ) prebuiltHrefs.get(i);
124             URL u = SearchIndex.getIndexableURL(index.getLocale(), href);
125             if (u == null) {
126                 // should never be here
127 docsToDelete.put(href, null);
128             }
129             if (newDocs.contains(u)) {
130                 // delete duplicates only
131 if (docsToDelete.get(href) != null) {
132                     // duplicates exist, leave map entry as is
133 } else {
134                     // no duplicates, do not delete
135 docsToDelete.remove(href);
136                 }
137             } else {
138                 // document should not be indexed at all (TOC not built)
139 // delete completely, not just duplicates
140 docsToDelete.put(href, null);
141             }
142         }
143         return docsToDelete;
144     }
145
146     /**
147      * Returns documents that must be deleted
148      */
149     private Map addNewDocuments(IProgressMonitor pm, Collection newDocs,
150             boolean opened) throws IndexingException {
151         Map prebuiltDocs = mergeIndexes(pm, opened);
152         checkCancelled(pm);
153         Collection docsToIndex = calculateDocsToAdd(newDocs, prebuiltDocs);
154         checkCancelled(pm);
155         Map docsToDelete = calculateNewToRemove(newDocs, prebuiltDocs);
156         pm.beginTask("", 10 * docsToIndex.size() + docsToDelete.size()); //$NON-NLS-1$
157 checkCancelled(pm);
158         addDocuments(new SubProgressMonitor(pm, 10 * docsToIndex.size()),
159                 docsToIndex, docsToDelete.size() == 0);
160         checkCancelled(pm);
161         removeNewDocuments(new SubProgressMonitor(pm, docsToDelete.size()),
162                 docsToDelete);
163         pm.done();
164         return docsToDelete;
165     }
166
167     private Collection calculateDocsToAdd(Collection newDocs, Map prebuiltDocs) {
168         // Calculate documents that were not in prebuilt indexes, and still need
169 // to be added
170 // (newDocs minus prebuiltDocs)
171 Collection docsToIndex = null;
172         if (prebuiltDocs.size() > 0) {
173             docsToIndex = new HashSet (newDocs);
174             for (Iterator it = prebuiltDocs.keySet().iterator(); it.hasNext();) {
175                 String href = (String ) it.next();
176                 URL u = SearchIndex.getIndexableURL(index.getLocale(), href);
177                 if (u != null) {
178                     docsToIndex.remove(u);
179                 }
180             }
181         } else {
182             docsToIndex = newDocs;
183         }
184         return docsToIndex;
185     }
186
187     /**
188      * @param docsToDelete
189      * Keys are /pluginid/href of all merged Docs. Values are null to
190      * delete href, or String[] of indexIds to delete duplicates with
191      * given index IDs
192      */
193     private void removeNewDocuments(IProgressMonitor pm, Map docsToDelete)
194             throws IndexingException {
195         pm = new LazyProgressMonitor(pm);
196         pm.beginTask("", docsToDelete.size()); //$NON-NLS-1$
197 checkCancelled(pm);
198         Set keysToDelete = docsToDelete.keySet();
199         if (keysToDelete.size() > 0) {
200             if (!index.beginRemoveDuplicatesBatch()) {
201                 throw new IndexingException();
202             }
203             MultiStatus multiStatus = null;
204             for (Iterator it = keysToDelete.iterator(); it.hasNext();) {
205                 String href = (String ) it.next();
206                 String [] indexIds = (String []) docsToDelete.get(href);
207                 if (indexIds == null) {
208                     // delete all copies
209 index.removeDocument(href);
210                     continue;
211                 }
212                 IStatus status = index.removeDuplicates(href, indexIds);
213                 if (status.getCode() != IStatus.OK) {
214                     if (multiStatus == null) {
215                         multiStatus = new MultiStatus(
216                                 HelpBasePlugin.PLUGIN_ID,
217                                 IStatus.WARNING,
218                                 "Some help documents could not removed from index.", //$NON-NLS-1$
219 null);
220                     }
221                     multiStatus.add(status);
222                 }
223                 checkCancelled(pm);
224                 pm.worked(1);
225                 if (multiStatus != null) {
226                     HelpBasePlugin.logStatus(multiStatus);
227                 }
228             }
229             if (!index.endRemoveDuplicatesBatch()) {
230                 throw new IndexingException();
231             }
232         }
233         pm.done();
234     }
235
236     private void addDocuments(IProgressMonitor pm, Collection addedDocs,
237             boolean lastOperation) throws IndexingException {
238         pm = new LazyProgressMonitor(pm);
239         // beginAddBatch()) called when processing prebuilt indexes
240 pm.beginTask("", addedDocs.size()); //$NON-NLS-1$
241 checkCancelled(pm);
242         pm.subTask(HelpBaseResources.UpdatingIndex);
243         MultiStatus multiStatus = null;
244         for (Iterator it = addedDocs.iterator(); it.hasNext();) {
245             URL doc = (URL ) it.next();
246             IStatus status = index.addDocument(getName(doc), doc);
247             if (status.getCode() != IStatus.OK) {
248                 if (multiStatus == null) {
249                     multiStatus = new MultiStatus(
250                             HelpBasePlugin.PLUGIN_ID,
251                             IStatus.ERROR,
252                             "Help documentation could not be indexed completely.", //$NON-NLS-1$
253 null);
254                 }
255                 multiStatus.add(status);
256             }
257             checkCancelled(pm);
258             pm.worked(1);
259         }
260         if (multiStatus != null) {
261             HelpBasePlugin.logStatus(multiStatus);
262         }
263         pm.subTask(HelpBaseResources.Writing_index);
264         if (!index.endAddBatch(addedDocs.size() > 0, lastOperation))
265             throw new IndexingException();
266         pm.done();
267     }
268
269     private void removeStaleDocuments(IProgressMonitor pm,
270             Collection removedDocs) throws IndexingException {
271         pm = new LazyProgressMonitor(pm);
272         pm.beginTask("", removedDocs.size()); //$NON-NLS-1$
273 pm.subTask(HelpBaseResources.Preparing_for_indexing);
274         checkCancelled(pm);
275
276         if (numRemoved > 0) {
277             if (!index.beginDeleteBatch()) {
278                 throw new IndexingException();
279             }
280             checkCancelled(pm);
281             pm.subTask(HelpBaseResources.UpdatingIndex);
282             MultiStatus multiStatus = null;
283             for (Iterator it = removedDocs.iterator(); it.hasNext();) {
284                 URL doc = (URL ) it.next();
285                 IStatus status = index.removeDocument(getName(doc));
286                 if (status.getCode() != IStatus.OK) {
287                     if (multiStatus == null) {
288                         multiStatus = new MultiStatus(
289                                 HelpBasePlugin.PLUGIN_ID,
290                                 IStatus.WARNING,
291                                 "Uninstalled or updated help documents could not be removed from index.", //$NON-NLS-1$
292 null);
293                     }
294                     multiStatus.add(status);
295                 }
296                 checkCancelled(pm);
297                 pm.worked(1);
298             }
299             if (multiStatus != null) {
300                 HelpBasePlugin.logStatus(multiStatus);
301             }
302             if (!index.endDeleteBatch()) {
303                 throw new IndexingException();
304             }
305         }
306         pm.done();
307     }
308
309     /**
310      * Returns the document identifier. Currently we use the document file name
311      * as identifier.
312      */
313     private String getName(URL doc) {
314         String name = doc.getFile();
315         // remove query string if any
316 int i = name.indexOf('?');
317         if (i != -1)
318             name = name.substring(0, i);
319         return name;
320     }
321
322     public class IndexingException extends Exception {
323         private static final long serialVersionUID = 1L;
324     }
325
326     /**
327      * Returns IDs of plugins which need docs added to index.
328      */
329     private Collection getAddedPlugins(SearchIndex index) {
330         // Get the list of added plugins
331 Collection addedPlugins = index.getDocPlugins().getAdded();
332
333         if (addedPlugins == null || addedPlugins.isEmpty())
334             return new ArrayList (0);
335         return addedPlugins;
336     }
337
338     /**
339      * Returns the documents to be added to index. The collection consists of
340      * the associated PluginURL objects.
341      */
342     private Collection getAddedDocuments(SearchIndex index) {
343         // Get the list of added plugins
344 Collection addedPlugins = getAddedPlugins(index);
345         // get the list of all navigation urls.
346 Set urls = getAllDocuments(index.getLocale());
347         Set addedDocs = new HashSet (urls.size());
348         for (Iterator docs = urls.iterator(); docs.hasNext();) {
349             String doc = (String ) docs.next();
350             // Assume the url is /pluginID/path_to_topic.html
351 int i = doc.indexOf('/', 1);
352             String plugin = i == -1 ? "" : doc.substring(1, i); //$NON-NLS-1$
353 if (!addedPlugins.contains(plugin)) {
354                 continue;
355             }
356
357             URL url = SearchIndex.getIndexableURL(index.getLocale(), doc);
358             if (url != null) {
359                 addedDocs.add(url);
360             }
361         }
362         //Add documents from global search participants
363 LuceneSearchParticipant[] participants = BaseHelpSystem.getLocalSearchManager().getGlobalParticipants();
364         for (int j=0; j<participants.length; j++) {
365             String participantId;
366             try {
367                 participantId = participants[j].getId();
368             }
369             catch (Throwable t) {
370                 // log the error and skip this participant
371 HelpBasePlugin.logError("Failed to get help search participant id for: " + participants[j].getClass().getName() + "; skipping this one.", t); //$NON-NLS-1$ //$NON-NLS-2$
372 continue;
373             }
374             Set set;
375             try {
376                 set = participants[j].getAllDocuments(index.getLocale());
377             }
378             catch (Throwable t) {
379                 // log the error and skip this participant
380 HelpBasePlugin.logError("Failed to retrieve documents from one of the help search participants: " + participants[j].getClass().getName() + "; skipping this one.", t); //$NON-NLS-1$ //$NON-NLS-2$
381 continue;
382             }
383             
384             for (Iterator docs = set.iterator(); docs.hasNext();) {
385                 String doc = (String ) docs.next();
386                 String id = null;
387                 int qloc = doc.indexOf('?');
388                 if (qloc!= -1) {
389                     String query = doc.substring(qloc+1);
390                     doc = doc.substring(0, qloc);
391                     HashMap arguments = new HashMap ();
392                     HelpURLConnection.parseQuery(query, arguments);
393                     id = (String )arguments.get("id"); //$NON-NLS-1$
394 }
395                 // Assume the url is /pluginID/path_to_topic.html
396 int i = doc.indexOf('/', 1);
397                 String plugin = i == -1 ? "" : doc.substring(1, i); //$NON-NLS-1$
398 if (!addedPlugins.contains(plugin)) {
399                     continue;
400                 }
401
402                 URL url = SearchIndex.getIndexableURL(index.getLocale(), doc, id, participantId);
403                 if (url != null) {
404                     addedDocs.add(url);
405                 }
406             }
407         }
408         return addedDocs;
409     }
410
411     /**
412      * Returns the documents to be removed from index. The collection consists
413      * of the associated PluginURL objects.
414      */
415     private Collection getRemovedDocuments(SearchIndex index) {
416         // Get the list of removed plugins
417 Collection removedPlugins = index.getDocPlugins().getRemoved();
418         if (removedPlugins == null || removedPlugins.isEmpty())
419             return new ArrayList (0);
420         // get the list of indexed docs. This is a hashtable (url, plugin)
421 HelpProperties indexedDocs = index.getIndexedDocs();
422         Set removedDocs = new HashSet (indexedDocs.size());
423         for (Iterator docs = indexedDocs.keySet().iterator(); docs.hasNext();) {
424             String doc = (String ) docs.next();
425             // Assume the url is /pluginID/path_to_topic.html
426 int i = doc.indexOf('/', 1);
427             String plugin = i == -1 ? "" : doc.substring(1, i); //$NON-NLS-1$
428 if (!removedPlugins.contains(plugin)) {
429                 continue;
430             }
431
432             URL url = SearchIndex.getIndexableURL(index.getLocale(), doc);
433             if (url != null) {
434                 removedDocs.add(url);
435             }
436         }
437         return removedDocs;
438     }
439
440     /**
441      * Adds the topic and its subtopics to the list of documents
442      */
443     private void add(ITopic topic, Set hrefs) {
444         String href = topic.getHref();
445         add(href, hrefs);
446         ITopic[] subtopics = topic.getSubtopics();
447         for (int i = 0; i < subtopics.length; i++)
448             add(subtopics[i], hrefs);
449     }
450     
451     private void add(String href, Set hrefs) {
452         if (href != null
453                 && !href.equals("") && !href.startsWith("http://") && !href.startsWith("https://")) //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
454 hrefs.add(href);
455     }
456
457     /**
458      * Returns the collection of href's for all the help topics.
459      */
460     private Set getAllDocuments(String locale) {
461         // Add documents from TOCs
462 HashSet hrefs = new HashSet ();
463         Toc[] tocs = index.getTocManager().getTocs(locale);
464         for (int i = 0; i < tocs.length; i++) {
465             ITopic[] topics = tocs[i].getTopics();
466             for (int j = 0; j < topics.length; j++) {
467                 add(topics[j], hrefs);
468             }
469             ITocContribution contrib = tocs[i].getTocContribution();
470             String [] extraDocs = contrib.getExtraDocuments();
471             for (int j=0;j<extraDocs.length;++j) {
472                 add(extraDocs[j], hrefs);
473             }
474             ITopic tocDescriptionTopic = tocs[i].getTopic(null);
475             if (tocDescriptionTopic != null)
476                 add(tocDescriptionTopic, hrefs);
477         }
478         return hrefs;
479     }
480
481     /**
482      * Obtains PluginIndexes pointing to prebuilt indexes
483      *
484      * @param pluginIds
485      * @param locale
486      * @return
487      */
488     private PrebuiltIndexes getIndexesToAdd(Collection pluginIds) {
489         PrebuiltIndexes indexes = new PrebuiltIndexes(index);
490         
491         IExtensionRegistry registry = Platform.getExtensionRegistry();
492         IConfigurationElement[] elements = registry.getConfigurationElementsFor(TocFileProvider.EXTENSION_POINT_ID_TOC);
493         
494         for (int i=0;i<elements.length;++i) {
495             IConfigurationElement elem = elements[i];
496             try {
497                 if (elem.getName().equals(ELEMENT_NAME_INDEX)) {
498                     String pluginId = elem.getNamespaceIdentifier();
499                     if (pluginIds.contains(pluginId)) {
500                         String path = elem.getAttribute(ATTRIBUTE_NAME_PATH);
501                         if (path != null) {
502                             indexes.add(pluginId, path);
503                         }
504                         else {
505                             String msg = "Element \"index\" in extension of \"org.eclipse.help.toc\" must specify a \"path\" attribute (plug-in: " + pluginId + ")"; //$NON-NLS-1$ //$NON-NLS-2$
506 HelpBasePlugin.logError(msg, null);
507                         }
508                     }
509                 }
510             }
511             catch (InvalidRegistryObjectException e) {
512                 // ignore this extension; move on
513 }
514         }
515         return indexes;
516     }
517
518     private Map mergeIndexes(IProgressMonitor monitor, boolean opened)
519             throws IndexingException {
520         Collection addedPluginIds = getAddedPlugins(index);
521         PrebuiltIndexes indexes = getIndexesToAdd(addedPluginIds);
522         PluginIndex[] pluginIndexes = indexes.getIndexes();
523         Map mergedDocs = null;
524         // Always perform add batch to ensure that index is created and saved
525 // even if no new documents
526 if (!index.beginAddBatch(opened)) {
527             throw new IndexingException();
528         }
529         if (pluginIndexes.length > 0) {
530             mergedDocs = index.merge(pluginIndexes, monitor);
531         }
532
533         if (mergedDocs == null) {
534             return Collections.EMPTY_MAP;
535         }
536         return mergedDocs;
537     }
538
539 }

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags