|
1
19
20 package org.netbeans.modules.java.source.usages;
21
22 import java.io.File  ;
23 import java.io.IOException ;
24 import java.text.ParseException ;
25 import java.util.Comparator ;
26 import java.util.EnumSet ;
27 import java.util.HashSet ;
28 import java.util.Iterator ;
29 import java.util.LinkedList ;
30 import java.util.List ;
31 import java.util.Map ;
32 import java.util.Set ;
33 import java.util.TreeSet ;
34 import java.util.concurrent.atomic.AtomicBoolean ;
35 import java.util.logging.Logger ;
36 import java.util.regex.Matcher ;
37 import java.util.regex.Pattern ;
38 import javax.lang.model.element.ElementKind;
39 import org.apache.lucene.analysis.KeywordAnalyzer;
40 import org.apache.lucene.document.Document;
41 import org.apache.lucene.index.IndexReader;
42 import org.apache.lucene.index.IndexWriter;
43 import org.apache.lucene.index.Term;
44 import org.apache.lucene.index.TermDocs;
45 import org.apache.lucene.index.TermEnum;
46 import org.apache.lucene.search.BooleanClause.Occur;
47 import org.apache.lucene.search.BooleanQuery;
48 import org.apache.lucene.search.BooleanQuery;
49 import org.apache.lucene.search.Hit;
50 import org.apache.lucene.search.Hits;
51 import org.apache.lucene.search.IndexSearcher;
52 import org.apache.lucene.search.Searcher;
53 import org.apache.lucene.search.Query;
54 import org.apache.lucene.search.TermQuery;
55 import org.apache.lucene.search.WildcardQuery;
56 import org.apache.lucene.store.Directory;
57 import org.apache.lucene.store.FSDirectory;
58 import org.apache.lucene.store.RAMDirectory;
59 import org.netbeans.api.java.source.ClassIndex;
60 import org.netbeans.modules.java.source.usages.LuceneIndexMBeanImpl;
61 import org.netbeans.modules.java.source.util.LowMemoryEvent;
62 import org.netbeans.modules.java.source.util.LowMemoryListener;
63 import org.netbeans.modules.java.source.util.LowMemoryNotifier;
64
65
69 class LuceneIndex extends Index {
70
71 private static final boolean debugIndexMerging = Boolean.getBoolean("LuceneIndex.debugIndexMerge"); private static final String REFERENCES = "refs";
74 private static final Logger LOGGER = Logger.getLogger(LuceneIndex.class.getName());
75
76 private final Directory directory;
77 private Long rootTimeStamp;
78
79 private IndexReader reader; private IndexWriter writer; private Set <String > rootPkgCache;
83 public static Index create (final File cacheRoot) throws IOException {
84 assert cacheRoot != null && cacheRoot.exists() && cacheRoot.canRead() && cacheRoot.canWrite();
85 return new LuceneIndex (getReferencesCacheFolder(cacheRoot));
86 }
87
88
89 private LuceneIndex (final File refCacheRoot) throws IOException {
90 assert refCacheRoot != null;
91 this.directory = FSDirectory.getDirectory(refCacheRoot, false);
92 }
93
94 @SuppressWarnings ("unchecked") public List <String > getUsagesData(final String resourceName, Set <ClassIndexImpl.UsageType> mask, BooleanOperator operator) throws IOException {
96 if (!isValid(false)) {
97 return null;
98 }
99 final Searcher searcher = new IndexSearcher (this.getReader());
100 try {
101 final List <String > result = new LinkedList <String > ();
102 Query query = null;
103 if (mask == null) {
104 query = new WildcardQuery(DocumentUtil.referencesTerm (resourceName,null));
105 }
106 else {
107 assert operator != null;
108 switch (operator) {
109 case AND:
110 query = new WildcardQuery(DocumentUtil.referencesTerm (resourceName, mask));
111 break;
112 case OR:
113 BooleanQuery booleanQuery = new BooleanQuery ();
114 for (ClassIndexImpl.UsageType ut : mask) {
115 final Query subQuery = new WildcardQuery(DocumentUtil.referencesTerm (resourceName, EnumSet.of(ut)));
116 booleanQuery.add(subQuery, Occur.SHOULD);
117 }
118 query = booleanQuery;
119 break;
120 default:
121 throw new IllegalArgumentException (operator.toString());
122 }
123 }
124 assert query != null;
125 final Hits hits = searcher.search (query);
126 for (Iterator <Hit> it = (Iterator <Hit>) hits.iterator(); it.hasNext();) {
127 final Hit hit = it.next ();
128 final Document doc = hit.getDocument();
129 final String user = DocumentUtil.getBinaryName(doc);
130 final String map = DocumentUtil.getRefereneType(doc,resourceName);
131 if (map != null) {
132 result.add (DocumentUtil.encodeUsage(user,map));
133 }
134 }
135 return result;
136 } finally {
137 searcher.close();
138 }
139 }
140
141 @SuppressWarnings ("unchecked") public List <String > getUsagesFQN(final String resourceName, final Set <ClassIndexImpl.UsageType>mask, final BooleanOperator operator) throws IOException {
143 if (!isValid(false)) {
144 return null;
145 }
146 assert resourceName != null;
147 assert mask != null;
148 assert operator != null;
149 final Searcher searcher = new IndexSearcher (this.getReader());
150 Query query;
151 try {
152 final List <String > result = new LinkedList <String > ();
153 switch (operator) {
154 case AND:
155 query = new WildcardQuery(DocumentUtil.referencesTerm (resourceName, mask));
156 break;
157 case OR:
158 BooleanQuery booleanQuery = new BooleanQuery ();
159 for (ClassIndexImpl.UsageType ut : mask) {
160 final Query subQuery = new WildcardQuery(DocumentUtil.referencesTerm (resourceName, EnumSet.of(ut)));
161 booleanQuery.add(subQuery, Occur.SHOULD);
162 }
163 query = booleanQuery;
164 break;
165 default:
166 throw new IllegalArgumentException (operator.toString());
167 }
168 final Hits hits = searcher.search (query);
169 for (Iterator <Hit> it = (Iterator <Hit>) hits.iterator(); it.hasNext();) {
170 final Hit hit = it.next ();
171 final Document doc = hit.getDocument();
172 final String user = DocumentUtil.getBinaryName(doc);
173 result.add (user);
174 }
175 return result;
176 } finally {
177 searcher.close();
178 }
179 }
180
181 public List <String > getReferencesData(final String resourceName) throws IOException {
182 if (!isValid(false)) {
183 return null;
184 }
185 Searcher searcher = new IndexSearcher (this.getReader());
186 try {
187 Hits hits = searcher.search(DocumentUtil.binaryNameQuery(resourceName));
188 assert hits.length() <= 1;
189 if (hits.length() == 0) {
190 return null;
191 }
192 else {
193 Hit hit = (Hit) hits.iterator().next();
194 return DocumentUtil.getReferences(hit.getDocument());
195 }
196 }
197 finally {
198 searcher.close();
199 }
200 }
201
202 @SuppressWarnings ("unchecked") public <T> void getDeclaredTypes (final String name, final ClassIndex.NameKind kind, final ResultConvertor<T> convertor, final Set <? super T> result) throws IOException {
204 if (!isValid(false)) {
205 LOGGER.fine(String.format("LuceneIndex[%s] is invalid!\n", this.toString()));
206 return;
207 }
208 assert name != null;
209 final Set <Term> toSearch = new TreeSet <Term> (new Comparator <Term>(){
210 public int compare (Term t1, Term t2) {
211 int ret = t1.field().compareTo(t2.field());
212 if (ret == 0) {
213 ret = t1.text().compareTo(t2.text());
214 }
215 return ret;
216 }
217 });
218 final IndexReader in = getReader();
219 switch (kind) {
220 case SIMPLE_NAME:
221 {
222 toSearch.add(DocumentUtil.simpleNameTerm(name));
223 break;
224 }
225 case PREFIX:
226 if (name.length() == 0) {
227 emptyPrefixSearch(in, convertor, result);
229 return;
230 }
231 else {
232 final Term nameTerm = DocumentUtil.simpleNameTerm(name);
233 prefixSearh(nameTerm, in, toSearch);
234 break;
235 }
236 case CASE_INSENSITIVE_PREFIX:
237 if (name.length() == 0) {
238 emptyPrefixSearch(in, convertor, result);
240 return;
241 }
242 else {
243 final Term nameTerm = DocumentUtil.caseInsensitiveNameTerm(name.toLowerCase()); prefixSearh(nameTerm, in, toSearch);
245 break;
246 }
247 case CAMEL_CASE:
248 if (name.length() == 0) {
249 throw new IllegalArgumentException ();
250 }
251 {
252 final StringBuilder patternString = new StringBuilder ();
253 char startChar = 0;
254 for (int i=0; i<name.length(); i++) {
255 char c = name.charAt(i);
256 if (i == 0) {
258 startChar = c;
259 }
260 patternString.append(c);
261 if (i == name.length()-1) {
262 patternString.append("\\w*"); }
264 else {
265 patternString.append("[\\p{Lower}\\p{Digit}]*"); }
267 }
268 final Pattern pattern = Pattern.compile(patternString.toString());
269 regExpSearch(pattern,DocumentUtil.simpleNameTerm(Character.toString(startChar)),in,toSearch);
270 break;
271 }
272 case CASE_INSENSITIVE_REGEXP:
273 if (name.length() == 0 || !Character.isJavaIdentifierStart(name.charAt(0))) {
274 throw new IllegalArgumentException ();
275 }
276 {
277 final Pattern pattern = Pattern.compile(name,Pattern.CASE_INSENSITIVE);
278 regExpSearch(pattern, DocumentUtil.caseInsensitiveNameTerm(name.toLowerCase()), in, toSearch); break;
280 }
281 case REGEXP:
282 if (name.length() == 0 || !Character.isJavaIdentifierStart(name.charAt(0))) {
283 throw new IllegalArgumentException ();
284 }
285 {
286 final Pattern pattern = Pattern.compile(name);
287 regExpSearch(pattern, DocumentUtil.simpleNameTerm(name), in, toSearch);
288 break;
289 }
290 default:
291 throw new UnsupportedOperationException (kind.toString());
292 }
293 TermDocs tds = in.termDocs();
294 LOGGER.fine(String.format("LuceneIndex.getDeclaredTypes[%s] returned %d elements\n",this.toString(), toSearch.size()));
295 final Iterator <Term> it = toSearch.iterator();
296 final ElementKind[] kindHolder = new ElementKind[1];
297 Set <Integer > docNums = new TreeSet <Integer >();
298 while (it.hasNext()) {
299 tds.seek(it.next());
300 while (tds.next()) {
301 docNums.add (tds.doc());
302 }
303 }
304 for (Integer docNum : docNums) {
305 final Document doc = in.document(docNum);
306 final String binaryName = DocumentUtil.getBinaryName(doc, kindHolder);
307 result.add (convertor.convert(kindHolder[0],binaryName));
308 }
309 }
310
311 private void regExpSearch (final Pattern pattern, final Term startTerm, final IndexReader in, final Set <Term> toSearch) throws IOException {
312 final String startText = startTerm.text();
313 final StringBuilder startBuilder = new StringBuilder ();
314 startBuilder.append(startText.charAt(0));
315 for (int i=1; i<startText.length(); i++) {
316 char c = startText.charAt(i);
317 if (!Character.isJavaIdentifierPart(c)) {
318 break;
319 }
320 startBuilder.append(c);
321 }
322 final String startPrefix = startBuilder.toString();
323 final String camelField = startTerm.field();
324 final TermEnum en = in.terms(startTerm);
325 try {
326 do {
327 Term term = en.term();
328 if (term != null && camelField == term.field() && term.text().startsWith(startPrefix)) {
329 final Matcher m = pattern.matcher(term.text());
330 if (m.matches()) {
331 toSearch.add (term);
332 }
333 }
334 else {
335 break;
336 }
337 } while (en.next());
338 } finally {
339 en.close();
340 }
341 }
342
343 private <T> void emptyPrefixSearch (final IndexReader in, final ResultConvertor<T> convertor, final Set <? super T> result) throws IOException {
344 final int bound = in.maxDoc();
345 final ElementKind[] kindHolder = new ElementKind[1];
346 for (int i=0; i<bound; i++) {
347 if (!in.isDeleted(i)) {
348 final Document doc = in.document(i);
349 if (doc != null) {
350 String binaryName = DocumentUtil.getBinaryName (doc, kindHolder);
351 if (binaryName == null) {
352 continue;
354 }
355 else {
356 result.add (convertor.convert(kindHolder[0],binaryName));
357 }
358 }
359 }
360 }
361 }
362
363 private void prefixSearh (Term nameTerm, final IndexReader in, final Set <Term> toSearch) throws IOException {
364 final String prefixField = nameTerm.field();
365 final String name = nameTerm.text();
366 final TermEnum en = in.terms(nameTerm);
367 try {
368 do {
369 Term term = en.term();
370 if (term != null && prefixField == term.field() && term.text().startsWith(name)) {
371 toSearch.add (term);
372 }
373 else {
374 break;
375 }
376 } while (en.next());
377 } finally {
378 en.close();
379 }
380 }
381
382
383 public void getPackageNames (final String prefix, final boolean directOnly, final Set <String > result) throws IOException {
384 if (!isValid(false)) {
385 return;
386 }
387 final IndexReader in = getReader();
388 final Term pkgTerm = DocumentUtil.packageNameTerm (prefix);
389 final String prefixField = pkgTerm.field();
390 if (prefix.length() == 0) {
391 if (directOnly && this.rootPkgCache != null) {
392 result.addAll(this.rootPkgCache);
393 }
394 else {
395 if (directOnly) {
396 this.rootPkgCache = new HashSet <String >();
397 }
398 final TermEnum terms = in.terms ();
399 try {
400 do {
401 final Term currentTerm = terms.term();
402 if (currentTerm != null && prefixField == currentTerm.field()) {
403 String pkgName = currentTerm.text();
404 if (directOnly) {
405 int index = pkgName.indexOf('.',prefix.length());
406 if (index>0) {
407 pkgName = pkgName.substring(0,index);
408 }
409 this.rootPkgCache.add(pkgName);
410 }
411 result.add(pkgName);
412 }
413 } while (terms.next());
414 } finally {
415 terms.close();
416 }
417 }
418 }
419 else {
420 final TermEnum terms = in.terms (pkgTerm);
421 try {
422 do {
423 final Term currentTerm = terms.term();
424 if (currentTerm != null && prefixField == currentTerm.field() && currentTerm.text().startsWith(prefix)) {
425 String pkgName = currentTerm.text();
426 if (directOnly) {
427 int index = pkgName.indexOf('.',prefix.length());
428 if (index>0) {
429 pkgName = pkgName.substring(0,index);
430 }
431 }
432 result.add(pkgName);
433 }
434 else {
435 break;
436 }
437 } while (terms.next());
438 } finally {
439 terms.close();
440 }
441 }
442 }
443
444 public boolean isUpToDate(String resourceName, long timeStamp) throws IOException {
445 if (!isValid(false)) {
446 return false;
447 }
448 try {
449 Searcher searcher = new IndexSearcher (this.getReader());
450 try {
451 Hits hits;
452 if (resourceName == null) {
453 synchronized (this) {
454 if (this.rootTimeStamp != null) {
455 return rootTimeStamp.longValue() >= timeStamp;
456 }
457 }
458 hits = searcher.search(new TermQuery(DocumentUtil.rootDocumentTerm()));
459 }
460 else {
461 hits = searcher.search(DocumentUtil.binaryNameQuery(resourceName));
462 }
463
464 assert hits.length() <= 1;
465 if (hits.length() == 0) {
466 return false;
467 }
468 else {
469 try {
470 Hit hit = (Hit) hits.iterator().next();
471 long cacheTime = DocumentUtil.getTimeStamp(hit.getDocument());
472 if (resourceName == null) {
473 synchronized (this) {
474 this.rootTimeStamp = new Long (cacheTime);
475 }
476 }
477 return cacheTime >= timeStamp;
478 } catch (ParseException pe) {
479 throw new IOException ();
480 }
481 }
482 } finally {
483 searcher.close();
484 }
485 } catch (java.io.FileNotFoundException fnf) {
486 this.clear();
487 return false;
488 }
489 }
490
491 public void store (final Map <String , List <String >> refs, final List <String > topLevels) throws IOException {
492 this.rootPkgCache = null;
493 boolean create = !isValid (false);
494 long timeStamp = System.currentTimeMillis();
495 if (!create) {
496 IndexReader in = getReader();
497 final Searcher searcher = new IndexSearcher (in);
498 try {
499 for (String topLevel : topLevels) {
500 Hits hits = searcher.search(DocumentUtil.binaryContentNameQuery(topLevel));
501 for (int i=0; i<hits.length(); i++) {
502 in.deleteDocument (hits.id(i));
503 }
504 }
505 in.deleteDocuments (DocumentUtil.rootDocumentTerm());
506 } finally {
507 searcher.close();
508 }
509 }
510 storeData(refs, create, timeStamp);
511 }
512
513 public void store(final Map <String , List <String >> refs, final Set <String > toDelete) throws IOException {
514 this.rootPkgCache = null;
515 boolean create = !isValid (false);
516 long timeStamp = System.currentTimeMillis();
517 if (!create) {
518 IndexReader in = getReader();
519 final Searcher searcher = new IndexSearcher (in);
520 try {
521 for (String toDeleteItem : toDelete) {
522 Hits hits = searcher.search(DocumentUtil.binaryNameQuery(toDeleteItem));
523 if (hits.length()>1) {
526 java.util.logging.Logger.getLogger("global").warning("Multiple index entries for binaryName: " + toDeleteItem); }
528 for (int i=0; i<hits.length(); i++) {
529 in.deleteDocument (hits.id(i));
530 }
531 }
532 in.deleteDocuments (DocumentUtil.rootDocumentTerm());
533 } finally {
534 searcher.close();
535 }
536 }
537 storeData(refs, create, timeStamp);
538 }
539
540 private void storeData (final Map <String , List <String >> refs, final boolean create, final long timeStamp) throws IOException {
541 IndexWriter out = getWriter(create);
542 if (debugIndexMerging) {
543 out.setInfoStream (System.err);
544 }
545 final LuceneIndexMBean indexSettings = LuceneIndexMBeanImpl.getDefault();
546 if (indexSettings != null) {
547 out.setMergeFactor(indexSettings.getMergeFactor());
548 out.setMaxMergeDocs(indexSettings.getMaxMergeDocs());
549 out.setMaxBufferedDocs(indexSettings.getMaxBufferedDocs());
550 }
551 LowMemoryNotifier lm = LowMemoryNotifier.getDefault();
552 LMListener lmListener = new LMListener ();
553 lm.addLowMemoryListener (lmListener);
554 Directory memDir = null;
555 IndexWriter activeOut = null;
556 if (lmListener.lowMemory.getAndSet(false)) {
557 activeOut = out;
558 }
559 else {
560 memDir = new RAMDirectory ();
561 activeOut = new IndexWriter (memDir,new KeywordAnalyzer(), true);
562 }
563 try {
564 activeOut.addDocument (DocumentUtil.createRootTimeStampDocument (timeStamp));
565 for (Iterator <Map.Entry <String ,List <String >>> it = refs.entrySet().iterator(); it.hasNext();) {
566 Map.Entry <String ,List <String >> refsEntry = it.next();
567 it.remove();
568 String cn = refsEntry.getKey();
569 List <String > cr = refsEntry.getValue();
570 Document newDoc = DocumentUtil.createDocument(cn,timeStamp,cr);
571 activeOut.addDocument(newDoc);
572 if (memDir != null && lmListener.lowMemory.getAndSet(false)) {
573 activeOut.close();
574 out.addIndexes(new Directory[] {memDir});
575 memDir = new RAMDirectory ();
576 activeOut = new IndexWriter (memDir,new KeywordAnalyzer(), true);
577 }
578 }
579 if (memDir != null) {
580 activeOut.close();
581 out.addIndexes(new Directory[] {memDir});
582 activeOut = null;
583 memDir = null;
584 }
585 synchronized (this) {
586 this.rootTimeStamp = new Long (timeStamp);
587 }
588 } finally {
589 lm.removeLowMemoryListener (lmListener);
590 }
591 }
592
593 public boolean isValid (boolean tryOpen) throws IOException {
594 boolean res = IndexReader.indexExists(this.directory);
595 if (res && tryOpen) {
596 try {
597 getReader();
598 } catch (java.io.FileNotFoundException e) {
599 res = false;
600 clear();
601 }
602 }
603 return res;
604 }
605
606 public synchronized void clear () throws IOException {
607 this.close ();
608 final String [] content = this.directory.list();
609 for (String file : content) {
610 directory.deleteFile(file);
611 }
612 }
613
614 public synchronized void close () throws IOException {
615 if (this.reader != null) {
616 this.reader.close();
617 this.reader = null;
618 }
619 if (this.writer != null) {
620 this.writer.close();
621 this.writer = null;
622 }
623 }
624
625 public @Override String toString () {
626 return this.directory.toString();
627 }
628
629 private synchronized IndexReader getReader () throws IOException {
630 if (this.reader == null) {
631 if (this.writer != null) {
632 this.writer.close();
633 this.writer = null;
634 }
635 this.reader = IndexReader.open(this.directory);
636 }
637 return this.reader;
638 }
639
640 private synchronized IndexWriter getWriter (final boolean create) throws IOException {
641 if (this.writer == null) {
642 if (this.reader != null) {
643 this.reader.close();
644 this.reader = null;
645 }
646 this.writer = new IndexWriter (this.directory,new KeywordAnalyzer(), create);
647 }
648 return this.writer;
649 }
650
651
652 private static File getReferencesCacheFolder (final File cacheRoot) throws IOException {
653 File refRoot = new File (cacheRoot,REFERENCES);
654 if (!refRoot.exists()) {
655 refRoot.mkdir();
656 }
657 return refRoot;
658 }
659
660 private static class LMListener implements LowMemoryListener {
661
662 private AtomicBoolean lowMemory = new AtomicBoolean (false);
663
664 public void lowMemory(LowMemoryEvent event) {
665 lowMemory.set(true);
666 }
667 }
668
669 }
670 |
Popular Tags |
Java API By Example, From Geeks To Geeks.