1 8 9 package com.sleepycat.je.evictor; 10 11 import java.text.NumberFormat ; 12 import java.util.ArrayList ; 13 import java.util.Iterator ; 14 import java.util.List ; 15 import java.util.logging.Level ; 16 import java.util.logging.Logger ; 17 18 import com.sleepycat.je.DatabaseException; 19 import com.sleepycat.je.EnvironmentStats; 20 import com.sleepycat.je.StatsConfig; 21 import com.sleepycat.je.config.EnvironmentParams; 22 import com.sleepycat.je.dbi.DatabaseImpl; 23 import com.sleepycat.je.dbi.DbConfigManager; 24 import com.sleepycat.je.dbi.DbTree; 25 import com.sleepycat.je.dbi.EnvironmentImpl; 26 import com.sleepycat.je.dbi.INList; 27 import com.sleepycat.je.dbi.MemoryBudget; 28 import com.sleepycat.je.latch.LatchSupport; 29 import com.sleepycat.je.log.LogManager; 30 import com.sleepycat.je.recovery.Checkpointer; 31 import com.sleepycat.je.tree.BIN; 32 import com.sleepycat.je.tree.IN; 33 import com.sleepycat.je.tree.Node; 34 import com.sleepycat.je.tree.SearchResult; 35 import com.sleepycat.je.tree.Tree; 36 import com.sleepycat.je.utilint.DaemonThread; 37 import com.sleepycat.je.utilint.DbLsn; 38 import com.sleepycat.je.utilint.TestHook; 39 import com.sleepycat.je.utilint.Tracer; 40 41 46 public class Evictor extends DaemonThread { 47 public static final String SOURCE_DAEMON = "daemon"; 48 public static final String SOURCE_MANUAL = "manual"; 49 public static final String SOURCE_CRITICAL = "critical"; 50 private static final boolean DEBUG = false; 51 52 private EnvironmentImpl envImpl; 53 private LogManager logManager; 54 private Level detailedTraceLevel; private volatile boolean active; 57 58 private IN nextNode; 59 60 61 private long currentRequiredEvictBytes; 62 63 64 private int nodesPerScan; 65 66 67 private long evictBytesSetting; 68 69 70 private boolean evictByLruOnly; 71 72 73 private NumberFormat formatter; 74 75 78 79 80 private int nEvictPasses = 0; 81 82 83 private long nNodesSelected = 0; 84 private long nNodesSelectedThisRun; 85 86 87 private int nNodesScanned = 0; 88 private int nNodesScannedThisRun; 89 90 94 private long nNodesEvicted = 0; 95 private long nNodesEvictedThisRun; 96 97 98 private long nBINsStripped = 0; 99 private long nBINsStrippedThisRun; 100 101 102 EvictProfile evictProfile; 103 private TestHook runnableHook; 104 105 public Evictor(EnvironmentImpl envImpl, String name) 106 throws DatabaseException { 107 108 super(0, name, envImpl); 109 this.envImpl = envImpl; 110 logManager = envImpl.getLogManager(); 111 nextNode = null; 112 113 DbConfigManager configManager = envImpl.getConfigManager(); 114 nodesPerScan = configManager.getInt 115 (EnvironmentParams.EVICTOR_NODES_PER_SCAN); 116 evictBytesSetting = configManager.getLong 117 (EnvironmentParams.EVICTOR_EVICT_BYTES); 118 evictByLruOnly = configManager.getBoolean 119 (EnvironmentParams.EVICTOR_LRU_ONLY); 120 detailedTraceLevel = Tracer.parseLevel 121 (envImpl, EnvironmentParams.JE_LOGGING_LEVEL_EVICTOR); 122 123 evictProfile = new EvictProfile(); 124 formatter = NumberFormat.getNumberInstance(); 125 126 active = false; 127 } 128 129 public String toString() { 130 StringBuffer sb = new StringBuffer (); 131 sb.append("<Evictor name=\"").append(name).append("\"/>"); 132 return sb.toString(); 133 } 134 135 139 public void addToQueue(Object o) 140 throws DatabaseException { 141 142 throw new DatabaseException 143 ("Evictor.addToQueue should never be called."); 144 } 145 146 149 public void loadStats(StatsConfig config, EnvironmentStats stat) 150 throws DatabaseException { 151 152 stat.setNEvictPasses(nEvictPasses); 153 stat.setNNodesSelected(nNodesSelected); 154 stat.setNNodesScanned(nNodesScanned); 155 stat.setNNodesExplicitlyEvicted(nNodesEvicted); 156 stat.setNBINsStripped(nBINsStripped); 157 stat.setRequiredEvictBytes(currentRequiredEvictBytes); 158 159 if (config.getClear()) { 160 nEvictPasses = 0; 161 nNodesSelected = 0; 162 nNodesScanned = 0; 163 nNodesEvicted = 0; 164 nBINsStripped = 0; 165 } 166 } 167 168 synchronized public void clearEnv() { 169 envImpl = null; 170 } 171 172 175 protected int nDeadlockRetries() 176 throws DatabaseException { 177 178 return envImpl.getConfigManager().getInt 179 (EnvironmentParams.EVICTOR_RETRY); 180 } 181 182 185 public void alert() { 186 if (!active) { 187 wakeup(); 188 } 189 } 190 191 194 public void onWakeup() 195 throws DatabaseException { 196 197 if (envImpl.isClosed()) { 198 return; 199 } 200 201 doEvict(SOURCE_DAEMON, 202 false, true); } 205 206 209 public void doEvict(String source) 210 throws DatabaseException { 211 212 doEvict(source, 213 false, true); } 216 217 221 private synchronized void doEvict(String source, 222 boolean criticalEviction, 223 boolean backgroundIO) 224 throws DatabaseException { 225 226 232 if (active) { 233 return; 234 } 235 active = true; 236 try { 237 238 242 boolean progress = true; 243 while (progress && 244 (criticalEviction || !isShutdownRequested()) && 245 isRunnable(source)) { 246 if (evictBatch 247 (source, backgroundIO, currentRequiredEvictBytes) == 0) { 248 progress = false; 249 } 250 } 251 } finally { 252 active = false; 253 } 254 } 255 256 259 public void doCriticalEviction(boolean backgroundIO) 260 throws DatabaseException { 261 262 MemoryBudget mb = envImpl.getMemoryBudget(); 263 long currentUsage = mb.getCacheMemoryUsage(); 264 long maxMem = mb.getCacheBudget(); 265 long over = currentUsage - maxMem; 266 267 if (over > mb.getCriticalThreshold()) { 268 if (DEBUG) { 269 System.out.println("***critical detected:" + over); 270 } 271 doEvict(SOURCE_CRITICAL, 272 true, backgroundIO); 274 } 275 } 276 277 285 long evictBatch(String source, 286 boolean backgroundIO, 287 long requiredEvictBytes) 288 throws DatabaseException { 289 290 nNodesSelectedThisRun = 0; 291 nNodesEvictedThisRun = 0; 292 nNodesScannedThisRun = 0; 293 nBINsStrippedThisRun = 0; 294 nEvictPasses++; 295 296 assert evictProfile.clear(); int nBatchSets = 0; 298 boolean finished = false; 299 long evictBytes = 0; 300 301 302 evictBytes += envImpl.getUtilizationTracker().evictMemory(); 303 304 INList inList = envImpl.getInMemoryINs(); 305 inList.latchMajor(); 306 int inListStartSize = inList.getSize(); 307 308 try { 309 310 316 if (inListStartSize == 0) { 317 nextNode = null; 318 return 0; 319 } else { 320 if (nextNode == null) { 321 nextNode = inList.first(); 322 } 323 } 324 325 ScanIterator scanIter = new ScanIterator(nextNode, inList); 326 327 336 while ((evictBytes < requiredEvictBytes) && 337 (nNodesScannedThisRun <= inListStartSize)) { 338 339 IN target = selectIN(inList, scanIter); 340 341 if (target == null) { 342 break; 343 } else { 344 assert evictProfile.count(target); evictBytes += evict 346 (inList, target, scanIter, backgroundIO); 347 } 348 nBatchSets++; 349 } 350 351 355 nextNode = scanIter.mark(); 356 finished = true; 357 358 } finally { 359 nNodesScanned += nNodesScannedThisRun; 360 inList.releaseMajorLatch(); 361 362 Logger logger = envImpl.getLogger(); 363 if (logger.isLoggable(detailedTraceLevel)) { 364 365 Tracer.trace(detailedTraceLevel, envImpl, 366 "Evictor: pass=" + nEvictPasses + 367 " finished=" + finished + 368 " source=" + source + 369 " requiredEvictBytes=" + 370 formatter.format(requiredEvictBytes) + 371 " evictBytes=" + 372 formatter.format(evictBytes) + 373 " inListSize=" + inListStartSize + 374 " nNodesScanned=" + nNodesScannedThisRun + 375 " nNodesSelected=" + nNodesSelectedThisRun + 376 " nEvicted=" + nNodesEvictedThisRun + 377 " nBINsStripped=" + nBINsStrippedThisRun + 378 " nBatchSets=" + nBatchSets); 379 } 380 } 381 382 assert LatchSupport.countLatchesHeld() == 0: "latches held = " + 383 LatchSupport.countLatchesHeld(); 384 return evictBytes; 385 } 386 387 390 boolean isRunnable(String source) 391 throws DatabaseException { 392 393 MemoryBudget mb = envImpl.getMemoryBudget(); 394 long currentUsage = mb.getCacheMemoryUsage(); 395 long maxMem = mb.getCacheBudget(); 396 boolean doRun = ((currentUsage - maxMem) > 0); 397 398 399 if (doRun) { 400 currentRequiredEvictBytes = 401 (currentUsage - maxMem) + evictBytesSetting; 402 if (DEBUG) { 403 if (source == SOURCE_CRITICAL) { 404 System.out.println("executed: critical runnable"); 405 } 406 } 407 } 408 409 410 if (runnableHook != null) { 411 doRun = ((Boolean ) runnableHook.getHookValue()).booleanValue(); 412 currentRequiredEvictBytes = maxMem; 413 } 414 415 419 Logger logger = envImpl.getLogger(); 420 if (logger.isLoggable(detailedTraceLevel)) { 421 422 428 Runtime r = Runtime.getRuntime(); 429 long totalBytes = r.totalMemory(); 430 long freeBytes= r.freeMemory(); 431 long usedBytes = r.totalMemory() - r.freeMemory(); 432 StringBuffer sb = new StringBuffer (); 433 sb.append(" source=").append(source); 434 sb.append(" doRun=").append(doRun); 435 sb.append(" JEusedBytes=").append(formatter.format(currentUsage)); 436 sb.append(" requiredEvict="). 437 append(formatter.format(currentRequiredEvictBytes)); 438 sb.append(" JVMtotalBytes= ").append(formatter.format(totalBytes)); 439 sb.append(" JVMfreeBytes= ").append(formatter.format(freeBytes)); 440 sb.append(" JVMusedBytes= ").append(formatter.format(usedBytes)); 441 logger.log(detailedTraceLevel, sb.toString()); 442 } 443 444 return doRun; 445 } 446 447 450 private IN selectIN(INList inList, ScanIterator scanIter) 451 throws DatabaseException { 452 453 454 IN target = null; 455 long targetGeneration = Long.MAX_VALUE; 456 int targetLevel = Integer.MAX_VALUE; 457 boolean targetDirty = true; 458 boolean envIsReadOnly = envImpl.isReadOnly(); 459 int scanned = 0; 460 boolean wrapped = false; 461 while (scanned < nodesPerScan) { 462 if (scanIter.hasNext()) { 463 IN in = scanIter.next(); 464 nNodesScannedThisRun++; 465 466 DatabaseImpl db = in.getDatabase(); 467 468 473 if (db == null || db.isDeleteFinished()) { 474 String inInfo = " IN type=" + in.getLogType() + " id=" + 475 in.getNodeId() + " not expected on INList"; 476 String errMsg = (db == null) ? inInfo : 477 "Database " + db.getDebugName() + " id=" + db.getId() + 478 inInfo; 479 throw new DatabaseException(errMsg); 480 } 481 482 483 if (db.isDeleted()) { 484 continue; 485 } 486 487 492 if (db.getId().equals(DbTree.ID_DB_ID)) { 493 continue; 494 } 495 496 502 if (envIsReadOnly && (target != null) && in.getDirty()) { 503 continue; 504 } 505 506 511 int evictType = in.getEvictionType(); 512 if (evictType == IN.MAY_NOT_EVICT) { 513 continue; 514 } 515 516 520 if (evictByLruOnly) { 521 522 526 if (targetGeneration > in.getGeneration()) { 527 targetGeneration = in.getGeneration(); 528 target = in; 529 } 530 } else { 531 532 536 int level = normalizeLevel(in, evictType); 537 if (targetLevel != level) { 538 if (targetLevel > level) { 539 targetLevel = level; 540 targetDirty = in.getDirty(); 541 targetGeneration = in.getGeneration(); 542 target = in; 543 } 544 } else if (targetDirty != in.getDirty()) { 545 if (targetDirty) { 546 targetDirty = false; 547 targetGeneration = in.getGeneration(); 548 target = in; 549 } 550 } else { 551 if (targetGeneration > in.getGeneration()) { 552 targetGeneration = in.getGeneration(); 553 target = in; 554 } 555 } 556 } 557 scanned++; 558 } else { 559 560 if (wrapped) { 561 break; 562 } else { 563 nextNode = inList.first(); 564 scanIter.reset(nextNode); 565 wrapped = true; 566 } 567 } 568 } 569 570 if (target != null) { 571 nNodesSelectedThisRun++; 572 nNodesSelected++; 573 } 574 return target; 575 } 576 577 592 public int normalizeLevel(IN in, int evictType) { 593 594 int level = in.getLevel() & IN.LEVEL_MASK; 595 596 if (level == 1 && evictType == IN.MAY_EVICT_LNS) { 597 level = 0; 598 } 599 600 return level; 601 } 602 603 607 private long evict(INList inList, 608 IN target, 609 ScanIterator scanIter, 610 boolean backgroundIO) 611 throws DatabaseException { 612 613 boolean envIsReadOnly = envImpl.isReadOnly(); 614 long evictedBytes = 0; 615 616 636 637 644 if (target.latchNoWait(false)) { 645 boolean targetIsLatched = true; 646 try { 647 if (target instanceof BIN) { 648 649 envImpl.lazyCompress(target); 650 651 656 evictedBytes = ((BIN) target).evictLNs(); 657 if (evictedBytes > 0) { 658 nBINsStrippedThisRun++; 659 nBINsStripped++; 660 } 661 } 662 663 671 if (evictedBytes == 0 && target.isEvictable()) { 672 673 Tree tree = target.getDatabase().getTree(); 674 675 676 targetIsLatched = false; 677 SearchResult result = 678 tree.getParentINForChildIN 679 (target, 680 true, false); if (result.exactParentFound) { 683 evictedBytes = evictIN(target, result.parent, 684 result.index, 685 inList, scanIter, 686 envIsReadOnly, 687 backgroundIO); 688 } 689 } 690 } finally { 691 if (targetIsLatched) { 692 target.releaseLatchIfOwner(); 693 } 694 } 695 } 696 697 return evictedBytes; 698 } 699 700 704 private long evictIN(IN child, 705 IN parent, 706 int index, 707 INList inlist, 708 ScanIterator scanIter, 709 boolean envIsReadOnly, 710 boolean backgroundIO) 711 throws DatabaseException { 712 713 long evictBytes = 0; 714 try { 715 assert parent.isLatchOwnerForWrite(); 716 717 long oldGenerationCount = child.getGeneration(); 718 719 724 IN renewedChild = (IN) parent.getTarget(index); 725 726 730 if ((renewedChild != null) && 731 (renewedChild.getGeneration() <= oldGenerationCount) && 732 renewedChild.latchNoWait(false)) { 733 734 try { 735 if (renewedChild.isEvictable()) { 736 737 741 long renewedChildLsn = DbLsn.NULL_LSN; 742 boolean newChildLsn = false; 743 if (renewedChild.getDirty()) { 744 if (!envIsReadOnly) { 745 boolean logProvisional = 746 isProvisionalRequired(renewedChild); 747 748 752 renewedChildLsn = renewedChild.log 753 (logManager, 754 false, logProvisional, 756 true, backgroundIO, 758 parent); 759 newChildLsn = true; 760 } 761 } else { 762 renewedChildLsn = parent.getLsn(index); 763 } 764 765 if (renewedChildLsn != DbLsn.NULL_LSN) { 766 767 scanIter.mark(); 768 inlist.removeLatchAlreadyHeld(renewedChild); 769 scanIter.resetToMark(); 770 771 evictBytes = renewedChild.getInMemorySize(); 772 if (newChildLsn) { 773 774 778 parent.updateEntry 779 (index, null, renewedChildLsn); 780 } else { 781 782 787 parent.updateEntry(index, (Node) null); 788 } 789 790 791 nNodesEvictedThisRun++; 792 nNodesEvicted++; 793 } 794 } 795 } finally { 796 renewedChild.releaseLatch(); 797 } 798 } 799 } finally { 800 parent.releaseLatch(); 801 } 802 803 return evictBytes; 804 } 805 806 809 private boolean isProvisionalRequired(IN target) { 810 811 818 if (target.getDatabase().isDeferredWrite()) { 819 return true; 820 } 821 822 826 Checkpointer ckpter = envImpl.getCheckpointer(); 827 if ((ckpter != null) && 828 (target.getLevel() < ckpter.getHighestFlushLevel())) { 829 return true; 830 } 831 832 return false; 833 } 834 835 838 IN getNextNode() { 839 return nextNode; 840 } 841 842 843 public void setRunnableHook(TestHook hook) { 844 runnableHook = hook; 845 } 846 847 848 static public class EvictProfile { 849 850 private List candidates = new ArrayList (); 851 852 853 public boolean count(IN target) { 854 candidates.add(new Long (target.getNodeId())); 855 return true; 856 } 857 858 public List getCandidates() { 859 return candidates; 860 } 861 862 public boolean clear() { 863 candidates.clear(); 864 return true; 865 } 866 } 867 868 873 private static class ScanIterator { 874 private INList inList; 875 private Iterator iter; 876 private IN nextMark; 877 878 ScanIterator(IN startingIN, INList inList) 879 throws DatabaseException { 880 881 this.inList = inList; 882 reset(startingIN); 883 } 884 885 void reset(IN startingIN) 886 throws DatabaseException { 887 888 iter = inList.tailSet(startingIN).iterator(); 889 } 890 891 IN mark() 892 throws DatabaseException { 893 894 if (iter.hasNext()) { 895 nextMark = (IN) iter.next(); 896 } else { 897 nextMark = (IN) inList.first(); 898 } 899 return (IN) nextMark; 900 } 901 902 void resetToMark() 903 throws DatabaseException { 904 905 reset(nextMark); 906 } 907 908 boolean hasNext() { 909 return iter.hasNext(); 910 } 911 912 IN next() { 913 return (IN) iter.next(); 914 } 915 916 void remove() { 917 iter.remove(); 918 } 919 } 920 } 921 | Popular Tags |