1 8 9 package com.sleepycat.je.cleaner; 10 11 import java.io.File ; 12 import java.io.IOException ; 13 import java.util.ArrayList ; 14 import java.util.HashSet ; 15 import java.util.Iterator ; 16 import java.util.List ; 17 import java.util.Set ; 18 19 import junit.framework.TestCase; 20 21 import com.sleepycat.je.CheckpointConfig; 22 import com.sleepycat.je.Cursor; 23 import com.sleepycat.je.Database; 24 import com.sleepycat.je.DatabaseConfig; 25 import com.sleepycat.je.DatabaseEntry; 26 import com.sleepycat.je.DatabaseException; 27 import com.sleepycat.je.DbInternal; 28 import com.sleepycat.je.DbTestProxy; 29 import com.sleepycat.je.Environment; 30 import com.sleepycat.je.EnvironmentConfig; 31 import com.sleepycat.je.LockMode; 32 import com.sleepycat.je.OperationStatus; 33 import com.sleepycat.je.Transaction; 34 import com.sleepycat.je.config.EnvironmentParams; 35 import com.sleepycat.je.dbi.CursorImpl; 36 import com.sleepycat.je.dbi.EnvironmentImpl; 37 import com.sleepycat.je.log.FileManager; 38 import com.sleepycat.je.tree.BIN; 39 import com.sleepycat.je.util.TestUtils; 40 import com.sleepycat.je.utilint.DbLsn; 41 42 public class FileSelectionTest extends TestCase { 43 44 private static final int DATA_SIZE = 100; 45 private static final int FILE_SIZE = 4096 * 10; 46 private static final int INITIAL_FILES = 5; 47 private static final byte[] MAIN_KEY_FOR_DUPS = {0, 1, 2, 3, 4, 5}; 48 49 private static final EnvironmentConfig envConfig = initConfig(); 50 private static final EnvironmentConfig highUtilizationConfig = 51 initConfig(); 52 private static final EnvironmentConfig steadyStateAutoConfig = 53 initConfig(); 54 static { 55 highUtilizationConfig.setConfigParam 56 (EnvironmentParams.CLEANER_MIN_UTILIZATION.getName(), 57 String.valueOf(90)); 58 59 steadyStateAutoConfig.setConfigParam 60 (EnvironmentParams.ENV_RUN_CLEANER.getName(), "true"); 61 } 62 63 static EnvironmentConfig initConfig() { 64 EnvironmentConfig config = TestUtils.initEnvConfig(); 65 DbInternal.disableParameterValidation(config); 66 config.setTransactional(true); 67 config.setAllowCreate(true); 68 config.setTxnNoSync(Boolean.getBoolean(TestUtils.NO_SYNC)); 69 config.setConfigParam(EnvironmentParams.LOG_FILE_MAX.getName(), 70 Integer.toString(FILE_SIZE)); 71 config.setConfigParam(EnvironmentParams.ENV_CHECK_LEAKS.getName(), 72 "false"); 73 config.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(), 74 "false"); 75 config.setConfigParam(EnvironmentParams.CLEANER_REMOVE.getName(), 76 "false"); 77 config.setConfigParam 78 (EnvironmentParams.ENV_RUN_CHECKPOINTER.getName(), "false"); 79 config.setConfigParam 80 (EnvironmentParams.CLEANER_MIN_FILES_TO_DELETE.getName(), "1"); 81 config.setConfigParam 82 (EnvironmentParams.CLEANER_LOCK_TIMEOUT.getName(), "1"); 83 config.setConfigParam 84 (EnvironmentParams.CLEANER_MAX_BATCH_FILES.getName(), "1"); 85 return config; 86 } 87 88 private static final CheckpointConfig forceConfig = new CheckpointConfig(); 89 static { 90 forceConfig.setForce(true); 91 } 92 93 private File envHome; 94 private Environment env; 95 private EnvironmentImpl envImpl; 96 private Database db; 97 private boolean dups; 98 99 100 private List firstKeysInFiles; 101 102 103 private Set existingKeys; 104 105 public FileSelectionTest() { 106 envHome = new File (System.getProperty(TestUtils.DEST_DIR)); 107 } 108 109 public void setUp() 110 throws IOException , DatabaseException { 111 112 TestUtils.removeLogFiles("Setup", envHome, false); 113 TestUtils.removeFiles("Setup", envHome, FileManager.DEL_SUFFIX); 114 } 115 116 public void tearDown() 117 throws IOException , DatabaseException { 118 119 try { 120 if (env != null) { 121 env.close(); 122 } 123 } catch (Throwable e) { 124 System.out.println("tearDown: " + e); 125 } 126 127 try { 129 TestUtils.removeLogFiles("tearDown", envHome, true); 130 TestUtils.removeFiles("tearDown", envHome, FileManager.DEL_SUFFIX); 131 } catch (Throwable e) { 132 System.out.println("tearDown: " + e); 133 } 134 136 db = null; 137 env = null; 138 envImpl = null; 139 envHome = null; 140 firstKeysInFiles = null; 141 } 142 143 private void openEnv() 144 throws DatabaseException { 145 146 openEnv(envConfig); 147 } 148 149 private void openEnv(EnvironmentConfig config) 150 throws DatabaseException { 151 152 env = new Environment(envHome, config); 153 envImpl = DbInternal.envGetEnvironmentImpl(env); 154 155 DatabaseConfig dbConfig = new DatabaseConfig(); 156 dbConfig.setTransactional(true); 157 dbConfig.setAllowCreate(true); 158 dbConfig.setSortedDuplicates(dups); 159 db = env.openDatabase(null, "cleanerFileSelection", dbConfig); 160 } 161 162 private void closeEnv() 163 throws DatabaseException { 164 165 if (db != null) { 166 db.close(); 167 db = null; 168 } 169 if (env != null) { 170 env.close(); 171 env = null; 172 } 173 } 174 175 178 public void testBaseline() 179 throws DatabaseException { 180 181 openEnv(); 182 writeData(); 183 verifyData(); 184 verifyDeletedFiles(null); 185 closeEnv(); 186 openEnv(); 187 verifyData(); 188 verifyDeletedFiles(null); 189 closeEnv(); 190 } 191 192 public void testBaselineDups() 193 throws DatabaseException { 194 195 dups = true; 196 testBaseline(); 197 } 198 199 202 public void testBasic() 203 throws DatabaseException { 204 205 openEnv(); 206 writeData(); 207 verifyDeletedFiles(null); 208 209 213 forceCleanOne(); 214 verifyDeletedFiles(new int[] {0}); 215 verifyData(); 216 217 221 int fileNum = INITIAL_FILES / 2; 222 int firstKey = ((Integer ) firstKeysInFiles.get(fileNum)).intValue(); 223 int nextKey = ((Integer ) firstKeysInFiles.get(fileNum + 1)).intValue(); 224 int count = nextKey - firstKey - 4; 225 deleteData(firstKey, count); 226 227 fileNum += 1; 228 firstKey = ((Integer ) firstKeysInFiles.get(fileNum)).intValue(); 229 nextKey = ((Integer ) firstKeysInFiles.get(fileNum + 1)).intValue(); 230 count = nextKey - firstKey - 4; 231 deleteData(firstKey, count); 232 233 forceCleanOne(); 234 forceCleanOne(); 235 verifyDeletedFiles(new int[] {0, fileNum - 1, fileNum}); 236 verifyData(); 237 238 closeEnv(); 239 } 240 241 public void testBasicDups() 242 throws DatabaseException { 243 244 dups = true; 245 testBasic(); 246 } 247 248 254 255 258 public void testCleaningMode() 259 throws DatabaseException { 260 261 int nextFile = -1; 262 int nCleaned; 263 264 268 openEnv(); 269 writeData(); 270 271 nCleaned = cleanRoutine(); 272 assertEquals(0, nCleaned); 273 nextFile = getNextDeletedFile(nextFile); 274 assertTrue(nextFile == -1); 275 276 verifyData(); 277 closeEnv(); 278 openEnv(); 279 verifyData(); 280 281 nCleaned = cleanRoutine(); 282 assertEquals(0, nCleaned); 283 nextFile = getNextDeletedFile(nextFile); 284 assertTrue(nextFile == -1); 285 286 verifyData(); 287 288 closeEnv(); 289 } 290 291 294 public void testRetry() 295 throws DatabaseException { 296 297 openEnv(highUtilizationConfig); 298 writeData(); 299 verifyData(); 300 301 305 int firstKey = ((Integer ) firstKeysInFiles.get(1)).intValue(); 306 int nextKey = ((Integer ) firstKeysInFiles.get(2)).intValue(); 307 int count = nextKey - firstKey - 1; 308 deleteData(firstKey, count); 309 verifyData(); 310 311 312 Transaction txn = env.beginTransaction(null, null); 313 Cursor cursor = db.openCursor(txn, null); 314 DatabaseEntry key = new DatabaseEntry(); 315 DatabaseEntry data = new DatabaseEntry(); 316 OperationStatus status; 317 if (dups) { 318 key.setData(MAIN_KEY_FOR_DUPS); 319 data.setData(TestUtils.getTestArray(nextKey - 1)); 320 status = cursor.getSearchBoth(key, data, LockMode.RMW); 321 } else { 322 key.setData(TestUtils.getTestArray(nextKey - 1)); 323 status = cursor.getSearchKey(key, data, LockMode.RMW); 324 } 325 assertEquals(OperationStatus.SUCCESS, status); 326 status = cursor.delete(); 327 assertEquals(OperationStatus.SUCCESS, status); 328 329 330 331 forceCleanOne(); 332 verifyDeletedFiles(null); 333 forceCleanOne(); 334 verifyDeletedFiles(null); 335 336 337 cursor.close(); 338 txn.abort(); 339 verifyData(); 340 341 342 forceCleanOne(); 343 verifyDeletedFiles(new int[] {0, 1, 2}); 344 verifyData(); 345 346 closeEnv(); 347 } 348 349 352 public void testMinFileUtilization() 353 throws DatabaseException { 354 355 356 EnvironmentConfig myConfig = initConfig(); 357 myConfig.setConfigParam 358 (EnvironmentParams.CLEANER_MIN_UTILIZATION.getName(), 359 String.valueOf(10)); 360 myConfig.setConfigParam 361 (EnvironmentParams.CLEANER_MIN_FILE_UTILIZATION.getName(), 362 String.valueOf(0)); 363 openEnv(myConfig); 364 365 366 writeData(); 367 verifyDeletedFiles(null); 368 int fileNum = INITIAL_FILES / 2; 369 int firstKey = ((Integer ) firstKeysInFiles.get(fileNum)).intValue(); 370 int nextKey = ((Integer ) firstKeysInFiles.get(fileNum + 1)).intValue(); 371 int count = ((nextKey - firstKey) * 2) / 3; 372 deleteData(firstKey, count); 373 374 375 env.cleanLog(); 376 env.checkpoint(forceConfig); 377 verifyDeletedFiles(null); 378 379 380 myConfig.setConfigParam 381 (EnvironmentParams.CLEANER_MIN_FILE_UTILIZATION.getName(), 382 String.valueOf(50)); 383 env.setMutableConfig(myConfig); 384 385 386 env.cleanLog(); 387 env.checkpoint(forceConfig); 388 verifyDeletedFiles(new int[] {fileNum}); 389 verifyData(); 390 391 closeEnv(); 392 } 393 394 private void printFiles(String msg) { 395 System.out.print(msg); 396 Long lastNum = envImpl.getFileManager().getLastFileNum(); 397 for (int i = 0; i <= (int) lastNum.longValue(); i += 1) { 398 String name = envImpl.getFileManager(). 399 getFullFileName(i, FileManager.JE_SUFFIX); 400 if (new File (name).exists()) { 401 System.out.print(" " + i); 402 } 403 } 404 System.out.println(""); 405 } 406 407 public void testRetryDups() 408 throws DatabaseException { 409 410 dups = true; 411 testRetry(); 412 } 413 414 418 public void testSteadyStateAutomatic() 419 throws DatabaseException { 420 421 doSteadyState(steadyStateAutoConfig, false, 13); 422 } 423 424 public void testSteadyStateAutomaticDups() 425 throws DatabaseException { 426 427 dups = true; 428 testSteadyStateAutomatic(); 429 } 430 431 434 public void testSteadyStateManual() 435 throws DatabaseException { 436 437 doSteadyState(envConfig, true, 13); 438 } 439 440 public void testSteadyStateManualDups() 441 throws DatabaseException { 442 443 dups = true; 444 testSteadyStateManual(); 445 } 446 447 450 public void testSteadyStateHighUtilization() 451 throws DatabaseException { 452 453 doSteadyState(highUtilizationConfig, true, 9); 454 } 455 456 public void testSteadyStateHighUtilizationDups() 457 throws DatabaseException { 458 459 dups = true; 460 testSteadyStateHighUtilization(); 461 } 462 463 472 private void doSteadyState(EnvironmentConfig config, 473 boolean manualCleaning, 474 int maxFileCount) 475 throws DatabaseException { 476 477 openEnv(config); 478 writeData(); 479 verifyData(); 480 481 final int iterations = 100; 482 483 for (int i = 0; i < iterations; i += 1) { 484 updateData(100, 100); 485 int cleaned = -1; 486 if (manualCleaning) { 487 cleaned = cleanRoutine(); 488 } else { 489 490 try { 491 Thread.sleep(25); 492 } catch (InterruptedException e) {} 493 } 494 495 499 env.checkpoint(forceConfig); 500 verifyData(); 501 int fileCount = 502 envImpl.getFileManager().getAllFileNumbers().length; 503 assertTrue("fileCount=" + fileCount + 504 " maxFileCount=" + maxFileCount, 505 fileCount <= maxFileCount); 506 if (false) { 507 System.out.println("fileCount=" + fileCount + 508 " cleaned=" + cleaned); 509 } 510 } 511 closeEnv(); 512 } 513 514 518 public void testTruncateDatabase() 519 throws DatabaseException { 520 521 int nCleaned; 522 523 openEnv(); 524 writeData(); 525 526 nCleaned = cleanRoutine(); 527 assertEquals(0, nCleaned); 528 529 db.truncate(null, false); 530 nCleaned = cleanRoutine(); 531 assertEquals(4, nCleaned); 532 533 closeEnv(); 534 } 535 536 539 public void testRemoveDatabase() 540 throws DatabaseException { 541 542 int nCleaned; 543 544 openEnv(); 545 writeData(); 546 547 String dbName = db.getDatabaseName(); 548 db.close(); 549 db = null; 550 551 nCleaned = cleanRoutine(); 552 assertEquals(0, nCleaned); 553 554 env.removeDatabase(null, dbName); 555 nCleaned = cleanRoutine(); 556 assertEquals(4, nCleaned); 557 558 closeEnv(); 559 } 560 561 public void testForceCleanFiles() 562 throws DatabaseException { 563 564 565 EnvironmentConfig myConfig = initConfig(); 566 openEnv(myConfig); 567 writeData(); 568 verifyData(); 569 env.cleanLog(); 570 env.checkpoint(forceConfig); 571 verifyDeletedFiles(null); 572 closeEnv(); 573 574 575 myConfig.setConfigParam 576 (EnvironmentParams.CLEANER_FORCE_CLEAN_FILES.getName(), 577 "3"); 578 openEnv(myConfig); 579 forceCleanOne(); 580 verifyDeletedFiles(new int[] {3}); 581 closeEnv(); 582 583 584 myConfig.setConfigParam 585 (EnvironmentParams.CLEANER_FORCE_CLEAN_FILES.getName(), 586 "0-1"); 587 openEnv(myConfig); 588 forceCleanOne(); 589 forceCleanOne(); 590 verifyDeletedFiles(new int[] {0, 1, 3}); 591 closeEnv(); 592 } 593 594 597 private void forceCleanOne() 598 throws DatabaseException { 599 600 envImpl.getCleaner().doClean(false, true); 603 env.checkpoint(forceConfig); 604 } 605 606 610 private int cleanRoutine() 611 throws DatabaseException { 612 613 return env.cleanLog(); 614 } 615 616 622 private void writeData() 623 throws DatabaseException { 624 625 DatabaseEntry key = new DatabaseEntry(); 626 DatabaseEntry data = new DatabaseEntry(new byte[DATA_SIZE]); 627 firstKeysInFiles = new ArrayList (); 628 existingKeys = new HashSet (); 629 630 Transaction txn = env.beginTransaction(null, null); 631 Cursor cursor = db.openCursor(txn, null); 632 int fileNum = -1; 633 634 for (int nextKey = 0; fileNum < INITIAL_FILES; nextKey += 1) { 635 636 OperationStatus status; 637 if (dups) { 638 key.setData(MAIN_KEY_FOR_DUPS); 639 data.setData(TestUtils.getTestArray(nextKey)); 640 status = cursor.putNoDupData(key, data); 641 } else { 642 key.setData(TestUtils.getTestArray(nextKey)); 643 data.setData(new byte[DATA_SIZE]); 644 status = cursor.putNoOverwrite(key, data); 645 } 646 647 assertEquals(OperationStatus.SUCCESS, status); 648 existingKeys.add(new Integer (nextKey)); 649 650 long lsn = getLsn(cursor); 651 if (DbLsn.getFileNumber(lsn) != fileNum) { 652 assertTrue(fileNum < DbLsn.getFileNumber(lsn)); 653 fileNum = (int) DbLsn.getFileNumber(lsn); 654 assertEquals(fileNum, firstKeysInFiles.size()); 655 firstKeysInFiles.add(new Integer (nextKey)); 656 } 657 } 658 659 cursor.close(); 660 txn.commit(); 661 env.checkpoint(forceConfig); 662 663 long firstActiveLsn = envImpl.getCheckpointer().getFirstActiveLsn(); 664 assertTrue(firstActiveLsn != DbLsn.NULL_LSN); 665 assertTrue(DbLsn.getFileNumber(firstActiveLsn) >= INITIAL_FILES); 666 } 667 668 671 private void deleteData(int firstKey, int keyCount) 672 throws DatabaseException { 673 674 DatabaseEntry key = new DatabaseEntry(); 675 DatabaseEntry data = new DatabaseEntry(); 676 677 Transaction txn = env.beginTransaction(null, null); 678 Cursor cursor = db.openCursor(txn, null); 679 680 for (int i = 0; i < keyCount; i += 1) { 681 int nextKey = firstKey + i; 682 OperationStatus status; 683 if (dups) { 684 key.setData(MAIN_KEY_FOR_DUPS); 685 data.setData(TestUtils.getTestArray(nextKey)); 686 status = cursor.getSearchBoth(key, data, null); 687 } else { 688 key.setData(TestUtils.getTestArray(nextKey)); 689 status = cursor.getSearchKey(key, data, null); 690 } 691 assertEquals(OperationStatus.SUCCESS, status); 692 status = cursor.delete(); 693 assertEquals(OperationStatus.SUCCESS, status); 694 existingKeys.remove(new Integer (nextKey)); 695 } 696 697 cursor.close(); 698 txn.commit(); 699 } 700 701 704 private void updateData(int firstKey, int keyCount) 705 throws DatabaseException { 706 707 DatabaseEntry key = new DatabaseEntry(); 708 DatabaseEntry data = new DatabaseEntry(); 709 710 Transaction txn = env.beginTransaction(null, null); 711 Cursor cursor = db.openCursor(txn, null); 712 713 for (int i = 0; i < keyCount; i += 1) { 714 int nextKey = firstKey + i; 715 OperationStatus status; 716 if (dups) { 717 key.setData(MAIN_KEY_FOR_DUPS); 718 data.setData(TestUtils.getTestArray(nextKey)); 719 status = cursor.getSearchBoth(key, data, null); 720 assertEquals(OperationStatus.SUCCESS, status); 721 assertEquals(MAIN_KEY_FOR_DUPS.length, key.getSize()); 722 assertEquals(nextKey, TestUtils.getTestVal(data.getData())); 723 } else { 724 key.setData(TestUtils.getTestArray(nextKey)); 725 status = cursor.getSearchKey(key, data, null); 726 assertEquals(OperationStatus.SUCCESS, status); 727 assertEquals(nextKey, TestUtils.getTestVal(key.getData())); 728 assertEquals(DATA_SIZE, data.getSize()); 729 } 730 status = cursor.putCurrent(data); 731 assertEquals(OperationStatus.SUCCESS, status); 732 } 733 734 cursor.close(); 735 txn.commit(); 736 } 737 738 741 private void verifyData() 742 throws DatabaseException { 743 744 DatabaseEntry key = new DatabaseEntry(); 745 DatabaseEntry data = new DatabaseEntry(); 746 747 Transaction txn = env.beginTransaction(null, null); 748 Cursor cursor = db.openCursor(txn, null); 749 750 for (Iterator i = existingKeys.iterator(); i.hasNext();) { 751 int nextKey = ((Integer ) i.next()).intValue(); 752 OperationStatus status; 753 if (dups) { 754 key.setData(MAIN_KEY_FOR_DUPS); 755 data.setData(TestUtils.getTestArray(nextKey)); 756 status = cursor.getSearchBoth(key, data, null); 757 assertEquals(OperationStatus.SUCCESS, status); 758 assertEquals(MAIN_KEY_FOR_DUPS.length, key.getSize()); 759 assertEquals(nextKey, TestUtils.getTestVal(data.getData())); 760 } else { 761 key.setData(TestUtils.getTestArray(nextKey)); 762 status = cursor.getSearchKey(key, data, null); 763 assertEquals(OperationStatus.SUCCESS, status); 764 assertEquals(nextKey, TestUtils.getTestVal(key.getData())); 765 assertEquals(DATA_SIZE, data.getSize()); 766 } 767 } 768 769 cursor.close(); 770 txn.commit(); 771 } 772 773 776 private void verifyDeletedFiles(int[] shouldNotExist) { 777 Long lastNum = envImpl.getFileManager().getLastFileNum(); 778 for (int i = 0; i <= (int) lastNum.longValue(); i += 1) { 779 boolean shouldExist = true; 780 if (shouldNotExist != null) { 781 for (int j = 0; j < shouldNotExist.length; j += 1) { 782 if (i == shouldNotExist[j]) { 783 shouldExist = false; 784 break; 785 } 786 } 787 } 788 String name = envImpl.getFileManager(). 789 getFullFileName(i, FileManager.JE_SUFFIX); 790 assertEquals(name, shouldExist, new File (name).exists()); 791 } 792 } 793 794 797 private int getNextDeletedFile(int afterFile) { 798 Long lastNum = envImpl.getFileManager().getLastFileNum(); 799 for (int i = afterFile + 1; i <= (int) lastNum.longValue(); i += 1) { 800 String name = envImpl.getFileManager(). 801 getFullFileName(i, FileManager.JE_SUFFIX); 802 if (!(new File (name).exists())) { 803 return i; 804 } 805 } 806 return -1; 807 } 808 809 812 private long getLsn(Cursor cursor) 813 throws DatabaseException { 814 815 CursorImpl impl = DbTestProxy.dbcGetCursorImpl(cursor); 816 BIN bin; 817 int index; 818 if (dups) { 819 bin = impl.getDupBIN(); 820 index = impl.getDupIndex(); 821 if (bin == null) { 822 bin = impl.getBIN(); 823 index = impl.getIndex(); 824 assertNotNull(bin); 825 } 826 } else { 827 assertNull(impl.getDupBIN()); 828 bin = impl.getBIN(); 829 index = impl.getIndex(); 830 assertNotNull(bin); 831 } 832 assertTrue(index >= 0); 833 long lsn = bin.getLsn(index); 834 assertTrue(lsn != DbLsn.NULL_LSN); 835 return lsn; 836 } 837 } 838 | Popular Tags |