1 17 package org.alfresco.repo.search.impl.lucene; 18 19 import java.io.File ; 20 import java.io.IOException ; 21 import java.util.Set ; 22 23 import org.alfresco.repo.search.IndexerException; 24 import org.alfresco.repo.search.transaction.LuceneIndexLock; 25 import org.alfresco.service.cmr.dictionary.DictionaryService; 26 import org.alfresco.service.cmr.repository.StoreRef; 27 import org.apache.log4j.Logger; 28 import org.apache.lucene.index.IndexReader; 29 import org.apache.lucene.index.IndexWriter; 30 import org.apache.lucene.index.MultiReader; 31 import org.apache.lucene.index.Term; 32 import org.apache.lucene.search.IndexSearcher; 33 import org.apache.lucene.search.Searcher; 34 import org.apache.lucene.store.Directory; 35 import org.apache.lucene.store.FSDirectory; 36 37 60 61 public abstract class LuceneBase implements Lockable 62 { 63 private static Logger s_logger = Logger.getLogger(LuceneBase.class); 64 65 68 69 private File baseDir; 70 71 74 75 private File deltaDir; 76 77 80 81 private File undoDir; 82 83 86 87 private IndexReader deltaReader; 88 89 92 93 private IndexWriter deltaWriter; 94 95 98 99 private IndexWriter mainWriter; 100 101 104 105 108 109 private IndexReader mainReader; 110 111 114 115 protected StoreRef store; 116 117 120 121 protected String deltaId; 122 123 private LuceneIndexLock luceneIndexLock; 124 125 private LuceneConfig config; 126 127 129 136 protected void initialise(StoreRef store, String deltaId, boolean createMain, boolean createDelta) 137 throws LuceneIndexException 138 { 139 this.store = store; 140 this.deltaId = deltaId; 141 142 String basePath = getMainPath(); 143 baseDir = new File (basePath); 144 if (createMain) 145 { 146 getWriteLock(); 147 } 148 try 149 { 150 try 151 { 152 initialiseFSDirectory(basePath, false, createMain).close(); 153 } 154 catch (IOException e) 155 { 156 s_logger.error("Error", e); 157 throw new LuceneIndexException("Failed to close directory after initialisation " + basePath); 158 } 159 if (deltaId != null) 160 { 161 String deltaPath = getDeltaPath(); 162 deltaDir = new File (deltaPath); 163 try 164 { 165 initialiseFSDirectory(deltaPath, createDelta, createDelta).close(); 166 } 167 catch (IOException e) 168 { 169 s_logger.error("Error", e); 170 throw new LuceneIndexException("Failed to close directory after initialisation " + deltaPath); 171 } 172 } 176 } 177 finally 178 { 179 if (createMain) 180 { 181 releaseWriteLock(); 182 } 183 } 184 } 185 186 191 private String getDeltaPath() 192 { 193 String deltaPath = getBasePath() + File.separator + "delta" + File.separator + this.deltaId + File.separator; 194 return deltaPath; 195 } 196 197 private String getMainPath() 198 { 199 String mainPath = getBasePath() + File.separator + "index" + File.separator; 200 return mainPath; 201 } 202 203 208 private String getBasePath() 209 { 210 if (config.getIndexRootLocation() == null) 211 { 212 throw new IndexerException("No configuration for index location"); 213 } 214 String basePath = config.getIndexRootLocation() 215 + File.separator + store.getProtocol() + File.separator + store.getIdentifier() + File.separator; 216 return basePath; 217 } 218 219 227 private Directory initialiseFSDirectory(String path, boolean deleteOnExit, boolean overwrite) 228 throws LuceneIndexException 229 { 230 try 231 { 232 File file = new File (path); 233 if (overwrite) 234 { 235 } 237 if (!file.exists()) 238 { 239 file.mkdirs(); 240 if (deleteOnExit) 241 { 242 file.deleteOnExit(); 243 } 244 245 return FSDirectory.getDirectory(file, true); 246 } 247 else 248 { 249 return FSDirectory.getDirectory(file, overwrite); 250 } 251 } 252 catch (IOException e) 253 { 254 s_logger.error("Error", e); 255 throw new LuceneIndexException("Filed to initialise lucene file directory " + path, e); 256 } 257 } 258 259 265 266 protected IndexSearcher getSearcher() throws LuceneIndexException 267 { 268 try 269 { 270 return new IndexSearcher(getMainPath()); 271 } 272 catch (IOException e) 273 { 274 s_logger.error("Error", e); 275 throw new LuceneIndexException("Failed to open IndexSarcher for " + getMainPath(), e); 276 } 277 } 278 279 protected Searcher getSearcher(LuceneIndexer luceneIndexer) throws LuceneIndexException 280 { 281 try 283 { 284 if (mainIndexExists()) 285 { 286 if (luceneIndexer == null) 287 { 288 return new IndexSearcher(getMainPath()); 289 } 290 else 291 { 292 luceneIndexer.flushPending(); 296 return new ClosingIndexSearcher(new MultiReader(new IndexReader[] { 297 new FilterIndexReaderByNodeRefs(IndexReader.open(getMainPath()), luceneIndexer 298 .getDeletions()), IndexReader.open(getDeltaPath()) })); 299 } 300 } 301 else 302 { 303 if (luceneIndexer == null) 304 { 305 return null; 306 } 307 else 308 { 309 luceneIndexer.flushPending(); 310 return new IndexSearcher(getDeltaPath()); 311 } 312 } 313 } 314 catch (IOException e) 315 { 316 s_logger.error("Error", e); 317 throw new LuceneIndexException("Failed to open IndexSarcher for " + getMainPath(), e); 318 } 319 } 320 321 327 328 protected IndexReader getDeltaReader() throws LuceneIndexException 329 { 330 if (deltaReader == null) 331 { 332 if (s_logger.isDebugEnabled()) 333 { 334 s_logger.debug("Trying to get index delta reader for tx " + deltaDir); 335 } 336 closeDeltaWriter(); 339 340 if (!indexExists(deltaDir)) 341 { 342 if (s_logger.isDebugEnabled()) 343 { 344 s_logger.debug("... index does not already exist for " + deltaDir + " creating ..."); 345 } 346 try 347 { 348 IndexWriter writer = new IndexWriter(deltaDir, new LuceneAnalyser(dictionaryService), true); 350 writer.setUseCompoundFile(true); 351 writer.close(); 352 if (s_logger.isDebugEnabled()) 353 { 354 s_logger.debug("... index created " + deltaDir); 355 } 356 } 357 catch (IOException e) 358 { 359 s_logger.error("Error", e); 360 throw new LuceneIndexException("Failed to create empty index for delta reader: " + deltaDir, e); 361 } 362 } 363 364 try 365 { 366 deltaReader = IndexReader.open(deltaDir); 367 if (s_logger.isDebugEnabled()) 368 { 369 s_logger.debug("Opened delta reader for " + deltaDir); 370 } 371 } 372 catch (IOException e) 373 { 374 s_logger.error("Error", e); 375 throw new LuceneIndexException("Failed to open delta reader: " + deltaDir, e); 376 } 377 378 } 379 return deltaReader; 380 } 381 382 private boolean indexExists(File dir) 383 { 384 return IndexReader.indexExists(dir); 385 } 386 387 392 393 protected void closeDeltaReader() throws LuceneIndexException 394 { 395 if (deltaReader != null) 396 { 397 try 398 { 399 try 400 { 401 deltaReader.close(); 402 if (s_logger.isDebugEnabled()) 403 { 404 s_logger.debug("Closed delta read for " + deltaDir); 405 } 406 } 407 catch (IOException e) 408 { 409 s_logger.error("Error", e); 410 throw new LuceneIndexException("Filed to close delta reader " + deltaDir, e); 411 } 412 } 413 finally 414 { 415 deltaReader = null; 416 } 417 } 418 419 } 420 421 427 protected IndexWriter getDeltaWriter() throws LuceneIndexException 428 { 429 if (deltaWriter == null) 430 { 431 if (s_logger.isDebugEnabled()) 432 { 433 s_logger.debug("Trying to create delta writer " + deltaDir); 434 } 435 closeDeltaReader(); 438 439 try 440 { 441 boolean create = !IndexReader.indexExists(deltaDir); 442 if (s_logger.isDebugEnabled()) 443 { 444 s_logger.debug("Creating delta writer " + deltaDir + " " + (create ? "CREATE" : "OPEN")); 445 } 446 deltaWriter = new IndexWriter(deltaDir, new LuceneAnalyser(dictionaryService), create); 447 } 448 catch (IOException e) 449 { 450 s_logger.error("Error", e); 451 throw new IndexerException("Failed to get delta writer for " + deltaDir, e); 452 } 453 } 454 deltaWriter.setUseCompoundFile(true); 455 deltaWriter.minMergeDocs = config.getIndexerMinMergeDocs(); 456 deltaWriter.mergeFactor = config.getIndexerMergeFactor(); 457 deltaWriter.maxMergeDocs = config.getIndexerMaxMergeDocs(); 458 if (s_logger.isDebugEnabled()) 459 { 460 s_logger.debug("Created delta writer " + deltaDir); 461 } 462 return deltaWriter; 463 } 464 465 470 471 protected void closeDeltaWriter() throws LuceneIndexException 472 { 473 if (deltaWriter != null) 474 { 475 if (s_logger.isDebugEnabled()) 476 { 477 s_logger.debug("Trying to close delta writer... " + deltaDir); 478 } 479 try 480 { 481 deltaWriter.close(); 483 if (s_logger.isDebugEnabled()) 484 { 485 s_logger.debug("Closed delta writer " + deltaDir); 486 } 487 } 488 catch (IOException e) 489 { 490 s_logger.error("Error", e); 491 throw new LuceneIndexException("Failed to close delta writer " + deltaDir, e); 492 } 493 finally 494 { 495 deltaWriter = null; 496 } 497 } 498 499 } 500 501 506 protected void saveDelta() throws LuceneIndexException 507 { 508 closeDeltaReader(); 511 closeDeltaWriter(); 512 } 513 514 520 protected void prepareToMergeIntoMain() throws LuceneIndexException 521 { 522 if (mainWriter != null) 523 { 524 throw new IndexerException("Can not merge as main writer is active"); 525 } 526 if (mainReader != null) 527 { 528 throw new IndexerException("Can not merge as main reader is active"); 529 } 530 if (s_logger.isDebugEnabled()) 531 { 532 s_logger.debug("Getting write lock for " + baseDir + " for " + deltaDir); 533 } 534 getWriteLock(); 535 if (s_logger.isDebugEnabled()) 536 { 537 s_logger.debug("Got write lock for " + baseDir + " for " + deltaDir); 538 } 539 try 540 { 541 getDeltaReader(); closeDeltaReader(); 543 } 544 catch (LuceneIndexException e) 545 { 546 s_logger.error("Error", e); 547 releaseWriteLock(); 548 throw e; 549 } 550 551 } 552 553 561 protected void mergeDeltaIntoMain(Set <Term> terms) throws LuceneIndexException 562 { 563 if (writeLockCount < 1) 564 { 565 throw new LuceneIndexException("Must hold the write lock to merge"); 566 } 567 568 try 569 { 570 if (!indexExists(baseDir)) 571 { 572 if (s_logger.isDebugEnabled()) 573 { 574 s_logger.debug("Creating base index " + baseDir); 575 } 576 try 577 { 578 mainWriter = new IndexWriter(baseDir, new LuceneAnalyser(dictionaryService), true); 579 mainWriter.setUseCompoundFile(true); 580 mainWriter.close(); 581 if (s_logger.isDebugEnabled()) 582 { 583 s_logger.debug("Created base index " + baseDir); 584 } 585 } 586 catch (IOException e) 587 { 588 s_logger.error("Error", e); 589 throw new LuceneIndexException("Failed to create empty base index at " + baseDir, e); 590 } 591 } 592 try 593 { 594 mainReader = IndexReader.open(baseDir); 595 if (s_logger.isDebugEnabled()) 596 { 597 s_logger.debug("Opened base index for deletes " + baseDir); 598 } 599 } 600 catch (IOException e) 601 { 602 s_logger.error("Error", e); 603 throw new LuceneIndexException("Failed to create base index reader at " + baseDir, e); 604 } 605 606 try 608 { 609 if ((mainReader.numDocs() > 0) && (terms.size() > 0)) 610 { 611 for (Term term : terms) 612 { 613 try 614 { 615 mainReader.delete(term); 616 } 617 catch (IOException e) 618 { 619 s_logger.error("Error", e); 620 throw new LuceneIndexException("Failed to delete term from main index at " + baseDir, e); 621 } 622 } 623 } 624 } 625 finally 626 { 627 try 628 { 629 try 630 { 631 mainReader.close(); 632 if (s_logger.isDebugEnabled()) 633 { 634 s_logger.debug("Completed index deletes on " + baseDir + " for " + deltaDir); 635 } 636 } 637 catch (IOException e) 638 { 639 s_logger.error("Error", e); 640 throw new LuceneIndexException("Failed to close from main index reader at " + baseDir, e); 641 } 642 } 643 finally 644 { 645 mainReader = null; 646 } 647 } 648 649 651 try 652 { 653 mainWriter = new IndexWriter(baseDir, new LuceneAnalyser(dictionaryService), false); 654 if (s_logger.isDebugEnabled()) 655 { 656 s_logger.debug("Opened index for append " + baseDir + " for " + deltaDir); 657 } 658 } 659 catch (IOException e) 660 { 661 s_logger.error("Error", e); 662 throw new LuceneIndexException("Failed to open main index for append at " + baseDir, e); 663 } 664 mainWriter.setUseCompoundFile(true); 665 666 mainWriter.minMergeDocs = config.getIndexerMinMergeDocs(); 667 mainWriter.mergeFactor = config.getIndexerMergeFactor(); 668 mainWriter.maxMergeDocs = config.getIndexerMaxMergeDocs(); 669 670 try 671 { 672 IndexReader reader = getDeltaReader(); 673 if (reader.numDocs() > 0) 674 { 675 IndexReader[] readers = new IndexReader[] { reader }; 676 try 677 { 678 mainWriter.mergeIndexes(readers); 679 } 681 catch (IOException e) 682 { 683 s_logger.error("Error", e); 684 throw new LuceneIndexException("Failed to merge indexes into the main index for " 685 + baseDir + " merging in " + deltaDir, e); 686 } 687 closeDeltaReader(); 689 } 690 else 691 { 692 closeDeltaReader(); 693 } 694 if (s_logger.isDebugEnabled()) 695 { 696 s_logger.debug("Closed index after append " + baseDir + " for " + deltaDir); 697 } 698 } 699 finally 700 { 701 try 702 { 703 try 704 { 705 mainWriter.close(); 706 } 707 catch (IOException e) 708 { 709 s_logger.error("Error", e); 710 throw new LuceneIndexException("Failed to cloase main index after append at " + baseDir, e); 711 } 712 } 713 finally 714 { 715 mainWriter = null; 716 } 717 } 718 } 719 finally 720 { 721 releaseWriteLock(); 722 } 723 } 724 725 733 protected void deleteDelta() throws LuceneIndexException 734 { 735 try 736 { 737 if (s_logger.isDebugEnabled()) 739 { 740 s_logger.debug("Deleting delta " + deltaDir); 741 } 742 try 743 { 744 closeDeltaReader(); 745 } 746 catch (LuceneIndexException e) 747 { 748 s_logger.warn(e); 749 } 750 try 751 { 752 closeDeltaWriter(); 753 } 754 catch (LuceneIndexException e) 755 { 756 s_logger.warn(e); 757 } 758 759 deltaDir = null; 768 769 if (mainReader != null) 771 { 772 try 773 { 774 mainReader.close(); 775 } 776 catch (IOException e) 777 { 778 s_logger.warn("Failed to close main reader", e); 779 } 780 } 781 mainReader = null; 782 783 if (mainWriter != null) 784 { 785 try 786 { 787 mainWriter.close(); 788 } 789 catch (IOException e) 790 { 791 s_logger.warn("Failed to close main writer", e); 792 } 793 } 794 mainWriter = null; 795 804 String deltaPath = getDeltaPath(); 806 File file = new File (deltaPath); 807 808 deleteDirectory(file); 809 } 810 finally 811 { 812 releaseWriteLock(); 813 } 814 } 815 816 821 private void deleteDirectory(File file) 822 { 823 File [] children = file.listFiles(); 824 if (children != null) 825 { 826 for (int i = 0; i < children.length; i++) 827 { 828 File child = children[i]; 829 if (child.isDirectory()) 830 { 831 deleteDirectory(child); 832 } 833 else 834 { 835 if (child.exists() && !child.delete() && child.exists()) 836 { 837 s_logger.warn("Failed to delete " + child); 838 } 839 } 840 } 841 } 842 if (file.exists() && !file.delete() && file.exists()) 843 { 844 s_logger.warn("Failed to delete " + file); 845 } 846 } 847 848 public LuceneIndexLock getLuceneIndexLock() 849 { 850 return luceneIndexLock; 851 } 852 853 public void setLuceneIndexLock(LuceneIndexLock luceneIndexLock) 854 { 855 this.luceneIndexLock = luceneIndexLock; 856 } 857 858 public void getReadLock() 859 { 860 getLuceneIndexLock().getReadLock(store); 861 } 862 863 public static boolean isWriteLocked(String directory) throws IOException 864 { 865 Directory dir = FSDirectory.getDirectory(directory, false); 866 boolean result = isWriteLocked(dir); 867 dir.close(); 868 return result; 869 } 870 871 public static boolean isWriteLocked(Directory directory) throws IOException 872 { 873 return directory.makeLock(IndexWriter.WRITE_LOCK_NAME).isLocked(); 874 875 } 876 877 private int writeLockCount = 0; 878 879 public void getWriteLock() throws LuceneIndexException 880 { 881 getLuceneIndexLock().getWriteLock(store); 882 writeLockCount++; 883 try 886 { 887 if (((writeLockCount == 1) && IndexReader.indexExists(baseDir) && (LuceneBase.isWriteLocked(baseDir 888 .getPath())))) 889 { 890 Directory dir = FSDirectory.getDirectory(baseDir, false); 891 try 892 { 893 dir.makeLock(IndexWriter.WRITE_LOCK_NAME).release(); 894 } 895 finally 896 { 897 dir.close(); 898 } 899 s_logger.warn("Releasing unexpected lucene index write lock for " + baseDir); 900 StackTraceElement [] trace = Thread.currentThread().getStackTrace(); 901 for (StackTraceElement el : trace) 902 { 903 s_logger.warn(el.toString()); 904 } 905 } 906 } 907 catch (IOException e) 908 { 909 s_logger.error("Error", e); 910 throw new LuceneIndexException("Write lock failed to check or clear any existing lucene locks", e); 911 } 912 } 913 914 public void releaseReadLock() 915 { 916 getLuceneIndexLock().releaseReadLock(store); 917 } 918 919 public void releaseWriteLock() 920 { 921 922 if (writeLockCount > 0) 923 { 924 try 925 { 926 if (((writeLockCount == 1) && IndexReader.indexExists(baseDir) && (LuceneBase.isWriteLocked(baseDir 927 .getPath())))) 928 { 929 Directory dir = FSDirectory.getDirectory(baseDir, false); 930 try 931 { 932 dir.makeLock(IndexWriter.WRITE_LOCK_NAME).release(); 933 } 934 finally 935 { 936 dir.close(); 937 } 938 } 939 } 940 catch (IOException e) 941 { 942 s_logger.error("Error", e); 943 throw new LuceneIndexException("Write lock failed to check or clear any existing lucene locks", e); 944 } 945 getLuceneIndexLock().releaseWriteLock(store); 946 writeLockCount--; 947 948 if (s_logger.isDebugEnabled()) 949 { 950 s_logger.debug("Released write lock " + baseDir + " for " + deltaDir); 951 } 952 } 953 } 954 955 private DictionaryService dictionaryService; 956 957 public boolean mainIndexExists() 958 { 959 return IndexReader.indexExists(baseDir); 960 } 961 962 protected IndexReader getReader() throws LuceneIndexException 963 { 964 965 if (!indexExists(baseDir)) 966 { 967 getWriteLock(); 968 try 969 { 970 if (!indexExists(baseDir)) 971 { 972 try 973 { 974 mainWriter = new IndexWriter(baseDir, new LuceneAnalyser(dictionaryService), true); 975 mainWriter.setUseCompoundFile(true); 976 mainWriter.close(); 977 mainWriter = null; 978 } 979 catch (IOException e) 980 { 981 s_logger.error("Error", e); 982 throw new LuceneIndexException("Failed to create empty main index", e); 983 } 984 } 985 } 986 finally 987 { 988 releaseWriteLock(); 989 } 990 } 991 992 try 993 { 994 return IndexReader.open(baseDir); 995 } 996 catch (IOException e) 997 { 998 s_logger.error("Error", e); 999 throw new LuceneIndexException("Failed to open main index reader", e); 1000 } 1001 1002 } 1003 1004 public void setDictionaryService(DictionaryService dictionaryService) 1005 { 1006 this.dictionaryService = dictionaryService; 1007 } 1008 1009 public DictionaryService getDictionaryService() 1010 { 1011 return dictionaryService; 1012 } 1013 1014 public void setLuceneConfig(LuceneConfig config) 1015 { 1016 this.config = config; 1017 } 1018 1019 public LuceneConfig getLuceneConfig() 1020 { 1021 return config; 1022 } 1023 1024 public String getDeltaId() 1025 { 1026 return deltaId; 1027 } 1028 1029} 1030 | Popular Tags |