Java > Open Source Codes > com > sleepycat > je > cleaner > UtilizationProfile


1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002,2006 Oracle. All rights reserved.
5  *
6  * $Id: UtilizationProfile.java,v 1.52 2006/11/28 13:52:05 mark Exp $
7  */
8
9 package com.sleepycat.je.cleaner;
10
11 import java.io.File ;
12 import java.util.ArrayList ;
13 import java.util.Arrays ;
14 import java.util.Iterator ;
15 import java.util.List ;
16 import java.util.Set ;
17 import java.util.SortedMap ;
18 import java.util.SortedSet ;
19 import java.util.StringTokenizer ;
20 import java.util.TreeMap ;
21 import java.util.logging.Level ;
22
23 import com.sleepycat.je.DatabaseConfig;
24 import com.sleepycat.je.DatabaseEntry;
25 import com.sleepycat.je.DatabaseException;
26 import com.sleepycat.je.DbInternal;
27 import com.sleepycat.je.OperationStatus;
28 import com.sleepycat.je.TransactionConfig;
29 import com.sleepycat.je.config.EnvironmentParams;
30 import com.sleepycat.je.dbi.CursorImpl;
31 import com.sleepycat.je.dbi.DatabaseId;
32 import com.sleepycat.je.dbi.DatabaseImpl;
33 import com.sleepycat.je.dbi.DbConfigManager;
34 import com.sleepycat.je.dbi.DbTree;
35 import com.sleepycat.je.dbi.EnvConfigObserver;
36 import com.sleepycat.je.dbi.EnvironmentImpl;
37 import com.sleepycat.je.dbi.MemoryBudget;
38 import com.sleepycat.je.dbi.CursorImpl.SearchMode;
39 import com.sleepycat.je.log.FileManager;
40 import com.sleepycat.je.log.entry.LNLogEntry;
41 import com.sleepycat.je.tree.BIN;
42 import com.sleepycat.je.tree.FileSummaryLN;
43 import com.sleepycat.je.tree.Tree;
44 import com.sleepycat.je.tree.TreeLocation;
45 import com.sleepycat.je.txn.AutoTxn;
46 import com.sleepycat.je.txn.BasicLocker;
47 import com.sleepycat.je.txn.LockType;
48 import com.sleepycat.je.txn.Locker;
49 import com.sleepycat.je.utilint.DbLsn;
50
51 /**
52  * The UP tracks utilization summary information for all log files.
53  *
54  * <p>Unlike the UtilizationTracker, the UP is not accessed under the log write
55  * latch and is instead synchronized on itself for protecting the cache. It is
56  * not accessed during the primary data access path, except for when flushing
57  * (writing) file summary LNs. This occurs in the following cases:
58  * <ol>
59  * <li>The summary information is flushed at the end of a checkpoint. This
60  * allows tracking to occur in memory in between checkpoints, and replayed
61  * during recovery.</li>
62  * <li>When committing the truncateDatabase and removeDatabase operations, the
63  * summary information is flushed because detail tracking for those operations
64  * is not replayed during recovery</li>
65  * <li>The evictor will ask the UtilizationTracker to flush the largest summary
66  * if the memory taken by the tracker exeeds its budget.</li>
67  * </ol>
68  *
69  * <p>The cache is populated by the RecoveryManager just before performing the
70  * initial checkpoint. The UP must be open and populated in order to respond
71  * to requests to flush summaries and to evict tracked detail, even if the
72  * cleaner is disabled.</p>
73  *
74  * <p>WARNING: While synchronized on this object, eviction is not permitted.
75  * If it were, this could cause deadlocks because the order of locking would be
76  * the UP object and then the evictor. During normal eviction the order is to
77  * first lock the evictor and then the UP, when evicting tracked detail.</p>
78  *
79  * <p>The methods in this class synchronize to protect the cached summary
80  * information. Some methods also access the UP database. However, because
81  * eviction must not occur while synchronized, UP database access is not
82  * performed while synchronized except in one case: when inserting a new
83  * summary record. In that case we disallow eviction during the database
84  * operation.</p>
85  */
86 public class UtilizationProfile implements EnvConfigObserver {
87
88     /*
89      * Note that age is a distance between files not a number of files, that
90      * is, deleted files are counted in the age.
91      */
92     private EnvironmentImpl env;
93     private UtilizationTracker tracker;
94     private DatabaseImpl fileSummaryDb; // stored fileNum -> FileSummary
95 private SortedMap fileSummaryMap; // cached fileNum -> FileSummary
96 private boolean cachePopulated;
97     private boolean rmwFixEnabled;
98
99     /**
100      * Minimum overall utilization threshold that triggers cleaning. Is
101      * non-private for unit tests.
102      */
103     int minUtilization;
104
105     /**
106      * Minimum utilization threshold for an individual log file that triggers
107      * cleaning. Is non-private for unit tests.
108      */
109     int minFileUtilization;
110
111     /**
112      * Minumum age to qualify for cleaning. If the first active LSN file is 5
113      * and the mininum age is 2, file 4 won't qualify but file 3 will. Must be
114      * greater than zero because we never clean the first active LSN file. Is
115      * non-private for unit tests.
116      */
117     int minAge;
118
119     /**
120      * An array of pairs of file numbers, where each pair is a range of files
121      * to be force cleaned. Index i is the from value and i+1 is the to value,
122      * both inclusive.
123      */
124     private long[] forceCleanFiles;
125
126     /**
127      * Creates an empty UP.
128      */
129     public UtilizationProfile(EnvironmentImpl env,
130                               UtilizationTracker tracker)
131         throws DatabaseException {
132
133         this.env = env;
134         this.tracker = tracker;
135         fileSummaryMap = new TreeMap ();
136
137         rmwFixEnabled = env.getConfigManager().getBoolean
138             (EnvironmentParams.CLEANER_RMW_FIX);
139         parseForceCleanFiles(env.getConfigManager().get
140             (EnvironmentParams.CLEANER_FORCE_CLEAN_FILES));
141
142         /* Initialize mutable properties and register for notifications. */
143         envConfigUpdate(env.getConfigManager());
144         env.addConfigObserver(this);
145     }
146
147     /**
148      * Process notifications of mutable property changes.
149      */
150     public void envConfigUpdate(DbConfigManager cm)
151         throws DatabaseException {
152
153         minAge = cm.getInt(EnvironmentParams.CLEANER_MIN_AGE);
154         minUtilization = cm.getInt(EnvironmentParams.CLEANER_MIN_UTILIZATION);
155         minFileUtilization = cm.getInt
156             (EnvironmentParams.CLEANER_MIN_FILE_UTILIZATION);
157     }
158
159     /**
160      * @see EnvironmentParams#CLEANER_RMW_FIX
161      * @see FileSummaryLN#postFetchInit
162      */
163     public boolean isRMWFixEnabled() {
164         return rmwFixEnabled;
165     }
166
167     /**
168      * Returns the number of files in the profile.
169      */
170     synchronized int getNumberOfFiles()
171         throws DatabaseException {
172
173         assert cachePopulated;
174
175         return fileSummaryMap.size();
176     }
177
178     /**
179      * Returns the cheapest file to clean from the given list of files. This
180      * method is used to select the first file to be cleaned in the batch of
181      * to-be-cleaned files.
182      */
183     synchronized Long getCheapestFileToClean(SortedSet files)
184         throws DatabaseException {
185
186         if (files.size() == 1) {
187             return (Long ) files.first();
188         }
189
190         assert cachePopulated;
191
192         Long bestFile = null;
193         int bestCost = Integer.MAX_VALUE;
194
195         for (Iterator iter = files.iterator(); iter.hasNext();) {
196             Long file = (Long ) iter.next();
197
198             /*
199              * Ignore files in the given set that are not in the profile. This
200              * can occur with multiple cleaner threads because we don't hold a
201              * synchronized lock during the entire execution of the
202              * FileSelector.selectFileForCleaning method. [#14431]
203              */
204             if (!fileSummaryMap.containsKey(file)) {
205                 continue;
206             }
207
208             /* Calculate this file's cost to clean. */
209             FileSummary summary = getFileSummary(file);
210             int thisCost = summary.getNonObsoleteCount();
211
212             /* Select this file if it has the lowest cost so far. */
213             if (bestFile == null || thisCost < bestCost) {
214                 bestFile = file;
215                 bestCost = thisCost;
216             }
217         }
218
219         return bestFile;
220     }
221
222     /**
223      * Returns the best file that qualifies for cleaning, or null if no file
224      * qualifies.
225      *
226      * @param fileSelector is used to determine valid cleaning candidates.
227      *
228      * @param forceCleaning is true to always select a file, even if its
229      * utilization is above the minimum utilization threshold.
230      *
231      * @param lowUtilizationFiles is a returned set of files that are below the
232      * minimum utilization threshold.
233      */
234     synchronized Long getBestFileForCleaning(FileSelector fileSelector,
235                                              boolean forceCleaning,
236                                              Set lowUtilizationFiles)
237         throws DatabaseException {
238
239         /* Start with an empty set.*/
240         if (lowUtilizationFiles != null) {
241             lowUtilizationFiles.clear();
242         }
243
244         assert cachePopulated;
245
246         /* Paranoia. There should always be 1 file. */
247         if (fileSummaryMap.size() == 0) {
248             return null;
249         }
250
251         /* Use local variables for mutable properties. */
252         final int useMinUtilization = minUtilization;
253         final int useMinFileUtilization = minFileUtilization;
254         final int useMinAge = minAge;
255
256         /* There must have been at least one checkpoint previously. */
257         long firstActiveLsn = env.getCheckpointer().getFirstActiveLsn();
258         if (firstActiveLsn == DbLsn.NULL_LSN) {
259             return null;
260         }
261
262         /* Calculate totals and find the best file. */
263         Iterator iter = fileSummaryMap.keySet().iterator();
264         Long bestFile = null;
265         int bestUtilization = 101;
266         long totalSize = 0;
267         long totalObsoleteSize = 0;
268
269         while (iter.hasNext()) {
270             Long file = (Long ) iter.next();
271             long fileNum = file.longValue();
272
273             /* Calculate this file's utilization. */
274             FileSummary summary = getFileSummary(file);
275             int obsoleteSize = summary.getObsoleteSize();
276
277             /*
278              * If the file is already being cleaned, only total the
279              * non-obsolete amount. This is an optimistic prediction of the
280              * results of cleaning, and is used to prevent over-cleaning.
281              * Update the total obsolete size to include the utilization DB
282              * records that will be deleted when the log file is deleted.
283              */
284             if (fileSelector.isFileCleaningInProgress(file)) {
285                 totalSize += summary.totalSize - obsoleteSize;
286                 totalObsoleteSize += estimateUPObsoleteSize(summary);
287                 continue;
288             }
289
290             /* Add this file's value to the totals. */
291             totalSize += summary.totalSize;
292             totalObsoleteSize += obsoleteSize;
293
294             /* If the file is too young to be cleaned, skip it. */
295             if (DbLsn.getFileNumber(firstActiveLsn) - fileNum < useMinAge) {
296                 continue;
297             }
298
299             /* Select this file if it has the lowest utilization so far. */
300             int thisUtilization = utilization(obsoleteSize, summary.totalSize);
301             if (bestFile == null || thisUtilization < bestUtilization) {
302                 bestFile = file;
303                 bestUtilization = thisUtilization;
304             }
305
306             /* Return all low utilization files. */
307             if (lowUtilizationFiles != null &&
308                 thisUtilization < useMinUtilization) {
309                 lowUtilizationFiles.add(file);
310             }
311         }
312
313         /*
314          * Return the best file if we are under the minimum utilization or
315          * we're cleaning aggressively.
316          */
317         int totalUtilization = utilization(totalObsoleteSize, totalSize);
318         if (forceCleaning ||
319             totalUtilization < useMinUtilization ||
320             bestUtilization < useMinFileUtilization) {
321             return bestFile;
322         } else {
323             return null;
324         }
325     }
326
327     /**
328      * Calculate the utilization percentage.
329      */
330     public static int utilization(long obsoleteSize, long totalSize) {
331         if (totalSize != 0) {
332             return (int) (((totalSize - obsoleteSize) * 100) / totalSize);
333         } else {
334             return 0;
335         }
336     }
337
338     /**
339      * Estimate the log size that will be made obsolete when a log file is
340      * deleted and we delete its UP records.
341      *
342      * Note that we do not count the space taken by the deleted FileSummaryLN
343      * records written during log file deletion. These add the same amount to
344      * the total log size and the obsolete log size, and therefore have a small
345      * impact on total utilization.
346      */
347     private int estimateUPObsoleteSize(FileSummary summary) {
348
349         /* Disabled for now; needs more testing. */
350         if (true) return 0;
351
352         /*
353          * FileSummaryLN overhead:
354          * 14 Header
355          * 8 Node
356          * 1 Deleted
357          * 4 Data Length (0)
358          * 32 Base Summary (8 X 4)
359          * 8 PackedOffsets size and length (2 * 4)
360          * 8 PackedOffsets first offset
361          */
362         final int OVERHEAD = 75;
363
364         /*
365          * Make an arbitrary estimate of the number of offsets per
366          * FileSummaryLN. Then estimate the total byte size, assuming all
367          * short (2 byte) offsets. Round up the total number of log entries.
368          */
369         int OFFSETS_PER_LN = 1000;
370         int BYTES_PER_LN = OVERHEAD + (OFFSETS_PER_LN * 2 /* Size of short */);
371         int totalNodes = summary.totalLNCount + summary.totalINCount;
372         int logEntries = (totalNodes / OFFSETS_PER_LN) + 1 /* Round up */;
373         return logEntries * BYTES_PER_LN;
374     }
375
376     /**
377      * Gets the base summary from the cached map. Add the tracked summary, if
378      * one exists, to the base summary. Sets all entries obsolete, if the file
379      * is in the forceCleanFiles set.
380      */
381     private synchronized FileSummary getFileSummary(Long file) {
382
383         /* Get base summary. */
384         FileSummary summary = (FileSummary) fileSummaryMap.get(file);
385         long fileNum = file.longValue();
386
387         /* Add tracked summary */
388         TrackedFileSummary trackedSummary = tracker.getTrackedFile(fileNum);
389         if (trackedSummary != null) {
390             FileSummary totals = new FileSummary();
391             totals.add(summary);
392             totals.add(trackedSummary);
393             summary = totals;
394         }
395
396         /* Set everything obsolete for a file in the forceCleanFiles set. */
397         if (isForceCleanFile(fileNum)) {
398             FileSummary allObsolete = new FileSummary();
399             allObsolete.add(summary);
400             allObsolete.obsoleteLNCount = allObsolete.totalLNCount;
401             allObsolete.obsoleteINCount = allObsolete.totalINCount;
402             summary = allObsolete;
403         }
404
405         return summary;
406     }
407
408     /**
409      * Returns whether the given file is in the forceCleanFiles set.
410      */
411     private boolean isForceCleanFile(long file) {
412
413         if (forceCleanFiles != null) {
414             for (int i = 0; i < forceCleanFiles.length; i += 2) {
415                 long from = forceCleanFiles[i];
416                 long to = forceCleanFiles[i + 1];
417                 if (file >= from && file <= to) {
418                     return true;
419                 }
420             }
421         }
422         return false;
423     }
424
425     /**
426      * Parses the je.cleaner.forceCleanFiles property value.
427      */
428     private void parseForceCleanFiles(String propValue) {
429
430         if (propValue == null || propValue.length() == 0) {
431             forceCleanFiles = null;
432         } else {
433
434             String errPrefix = "Error in " +
435                 EnvironmentParams.CLEANER_FORCE_CLEAN_FILES.getName() +
436                 "=" + propValue + ": ";
437
438             StringTokenizer tokens = new StringTokenizer
439                 (propValue, ",-", true /*returnDelims*/);
440
441             /* Resulting list of Long file numbers. */
442             List list = new ArrayList ();
443
444             while (tokens.hasMoreTokens()) {
445
446                 /* Get "from" file number. */
447                 String fromStr = tokens.nextToken();
448                 long fromNum;
449                 try {
450                     fromNum = Long.parseLong(fromStr, 16);
451                 } catch (NumberFormatException e) {
452                     throw new IllegalArgumentException
453                         (errPrefix + "Invalid hex file number: " + fromStr);
454                 }
455
456                 long toNum = -1;
457                 if (tokens.hasMoreTokens()) {
458
459                     /* Get delimiter. */
460                     String delim = tokens.nextToken();
461                     if (",".equals(delim)) {
462                         toNum = fromNum;
463                     } else if ("-".equals(delim)) {
464
465                         /* Get "to" file number." */
466                         if (tokens.hasMoreTokens()) {
467                             String toStr = tokens.nextToken();
468                             try {
469                                 toNum = Long.parseLong(toStr, 16);
470                             } catch (NumberFormatException e) {
471                                 throw new IllegalArgumentException
472                                     (errPrefix + "Invalid hex file number: " +
473                                      toStr);
474                             }
475                         } else {
476                             throw new IllegalArgumentException
477                                 (errPrefix + "Expected file number: " + delim);
478                         }
479                     } else {
480                         throw new IllegalArgumentException
481                             (errPrefix + "Expected '-' or ',': " + delim);
482                     }
483                 } else {
484                     toNum = fromNum;
485                 }
486
487                 assert toNum != -1;
488                 list.add(new Long (fromNum));
489                 list.add(new Long (toNum));
490             }
491
492             forceCleanFiles = new long[list.size()];
493             for (int i = 0; i < forceCleanFiles.length; i += 1) {
494                 forceCleanFiles[i] = ((Long ) list.get(i)).longValue();
495             }
496         }
497     }
498
499     /**
500      * Count the given tracked info as obsolete and then log the summaries.
501      */
502     public void countAndLogSummaries(TrackedFileSummary[] summaries)
503         throws DatabaseException {
504
505         /* Count tracked info under the log write latch. */
506         env.getLogManager().countObsoleteNodes(summaries);
507
508         /* Utilization flushing may be disabled for unittests. */
509         if (!DbInternal.getCheckpointUP
510         (env.getConfigManager().getEnvironmentConfig())) {
511             return;
512         }
513
514         /* Write out the modified file summaries. */
515         for (int i = 0; i < summaries.length; i += 1) {
516             long fileNum = summaries[i].getFileNumber();
517             TrackedFileSummary tfs = tracker.getTrackedFile(fileNum);
518             if (tfs != null) {
519                 flushFileSummary(tfs);
520             }
521         }
522     }
523
524     /**
525      * Returns a copy of the current file summary map, optionally including
526      * tracked summary information, for use by the DbSpace utility and by unit
527      * tests. The returned map's key is a Long file number and its value is a
528      * FileSummary.
529      */
530     public synchronized SortedMap getFileSummaryMap(boolean includeTrackedFiles)
531         throws DatabaseException {
532
533         assert cachePopulated;
534
535         if (includeTrackedFiles) {
536             TreeMap map = new TreeMap ();
537             Iterator iter = fileSummaryMap.keySet().iterator();
538             while (iter.hasNext()) {
539                 Long file = (Long ) iter.next();
540                 FileSummary summary = getFileSummary(file);
541                 map.put(file, summary);
542             }
543             TrackedFileSummary[] trackedFiles = tracker.getTrackedFiles();
544             for (int i = 0; i < trackedFiles.length; i += 1) {
545                 TrackedFileSummary summary = trackedFiles[i];
546                 long fileNum = summary.getFileNumber();
547                 Long file = new Long (fileNum);
548                 if (!map.containsKey(file)) {
549                     map.put(file, summary);
550                 }
551             }
552             return map;
553         } else {
554             return new TreeMap (fileSummaryMap);
555         }
556     }
557
558     /**
559      * Clears the cache of file summary info. The cache starts out unpopulated
560      * and is populated on the first call to getBestFileForCleaning.
561      */
562     public synchronized void clearCache() {
563
564         int memorySize = fileSummaryMap.size() *
565             MemoryBudget.UTILIZATION_PROFILE_ENTRY;
566         MemoryBudget mb = env.getMemoryBudget();
567         mb.updateMiscMemoryUsage(0 - memorySize);
568
569         fileSummaryMap = new TreeMap ();
570         cachePopulated = false;
571     }
572
573     /**
574      * Removes a file from the utilization database and the profile, after it
575      * has been deleted by the cleaner.
576      */
577     void removeFile(Long fileNum)
578         throws DatabaseException {
579
580         /* Synchronize to update the cache. */
581         synchronized (this) {
582             assert cachePopulated;
583
584             /* Remove from the cache. */
585             if (fileSummaryMap.remove(fileNum) != null) {
586                 MemoryBudget mb = env.getMemoryBudget();
587                 mb.updateMiscMemoryUsage
588                     (0 - MemoryBudget.UTILIZATION_PROFILE_ENTRY);
589             }
590         }
591
592         /* Do not synchronize during LN deletion, to permit eviction. */
593         deleteFileSummary(fileNum);
594     }
595
596     /**
597      * For the LN at the cursor position deletes all LNs for the file. This
598      * method performs eviction and is not synchronized.
599      */
600     private void deleteFileSummary(Long fileNum)
601         throws DatabaseException {
602
603         Locker locker = null;
604         CursorImpl cursor = null;
605         try {
606             locker = new BasicLocker(env);
607             cursor = new CursorImpl(fileSummaryDb, locker);
608             /* Perform eviction in unsynchronized methods. */
609             cursor.setAllowEviction(true);
610
611             DatabaseEntry keyEntry = new DatabaseEntry();
612             DatabaseEntry dataEntry = new DatabaseEntry();
613             long fileNumVal = fileNum.longValue();
614
615             /* Search by file number. */
616             if (!getFirstFSLN
617                 (cursor, fileNumVal, keyEntry, dataEntry, LockType.WRITE)) {
618                 return;
619             }
620
621             /* Delete all LNs for this file number. */
622             OperationStatus status = OperationStatus.SUCCESS;
623             while (status == OperationStatus.SUCCESS) {
624
625                 /* Perform eviction once per operation. */
626                 env.getEvictor().doCriticalEviction(true); // backgroundIO
627
628                 FileSummaryLN ln = (FileSummaryLN)
629                     cursor.getCurrentLN(LockType.NONE);
630
631                 if (ln != null) {
632                     /* Stop if the file number changes. */
633                     if (fileNumVal != ln.getFileNumber(keyEntry.getData())) {
634                         break;
635                     }
636
637                     TrackedFileSummary tfs =
638                         tracker.getTrackedFile(fileNumVal);
639                     /* Associate the tracked summary so it will be cleared. */
640                     if (tfs != null) {
641                         ln.setTrackedSummary(tfs);
642                     }
643
644                     /*
645                      * Do not evict after deleting since the compressor would
646                      * have to fetch it again.
647                      */
648             cursor.latchBIN();
649             try {
650             cursor.delete();
651             } finally {
652             cursor.releaseBIN();
653             }
654                 }
655
656                 status = cursor.getNext
657                     (keyEntry, dataEntry, LockType.WRITE,
658                      true, // forward
659 false); // alreadyLatched
660 }
661         } finally {
662             if (cursor != null) {
663                 cursor.releaseBINs();
664                 cursor.close();
665             }
666             if (locker != null) {
667                 locker.operationEnd();
668             }
669         }
670     }
671
672     /**
673      * Updates and stores the FileSummary for a given tracked file, if flushing
674      * of the summary is allowed.
675      */
676     public void flushFileSummary(TrackedFileSummary tfs)
677         throws DatabaseException {
678
679         if (tfs.getAllowFlush()) {
680             putFileSummary(tfs);
681         }
682     }
683
684     /**
685      * Updates and stores the FileSummary for a given tracked file. This
686      * method is synchronized and may not perform eviction.
687      */
688     private synchronized PackedOffsets putFileSummary(TrackedFileSummary tfs)
689         throws DatabaseException {
690
691         if (env.isReadOnly()) {
692             throw new DatabaseException
693                 ("Cannot write file summary in a read-only environment");
694         }
695
696         if (tfs.isEmpty()) {
697             return null; // no delta
698 }
699
700         if (!cachePopulated) {
701             /* Db does not exist and this is a read-only environment. */
702             return null;
703         }
704
705         long fileNum = tfs.getFileNumber();
706         Long fileNumLong = new Long (fileNum);
707
708         /* Get existing file summary or create an empty one. */
709         FileSummary summary = (FileSummary) fileSummaryMap.get(fileNumLong);
710         if (summary == null) {
711
712             /*
713              * An obsolete node may have been counted after its file was
714              * deleted, for example, when compressing a BIN. Do not insert
715              * a new profile record if no corresponding log file exists.
716              */
717             File file = new File
718                 (env.getFileManager().getFullFileName
719                     (fileNum, FileManager.JE_SUFFIX));
720             if (!file.exists()) {
721                 return null;
722             }
723
724             summary = new FileSummary();
725         }
726
727         /*
728          * The key discriminator is a sequence that must be increasing over the
729          * life of the file. We use the sum of all entries counted. We must
730          * add the tracked and current summaries here to calculate the key.
731          */
732         FileSummary tmp = new FileSummary();
733         tmp.add(summary);
734         tmp.add(tfs);
735         int sequence = tmp.getEntriesCounted();
736
737         /* Insert an LN with the existing and tracked summary info. */
738         FileSummaryLN ln = new FileSummaryLN(summary);
739         ln.setTrackedSummary(tfs);
740         insertFileSummary(ln, fileNum, sequence);
741
742         /* Cache the updated summary object. */
743         summary = ln.getBaseSummary();
744         if (fileSummaryMap.put(fileNumLong, summary) == null) {
745             MemoryBudget mb = env.getMemoryBudget();
746             mb.updateMiscMemoryUsage
747                 (MemoryBudget.UTILIZATION_PROFILE_ENTRY);
748         }
749
750         return ln.getObsoleteOffsets();
751     }
752
753     /**
754      * Returns the stored/packed obsolete offsets and the tracked obsolete
755      * offsets for the given file. The tracked summary object returned can be
756      * used to test for obsolete offsets that are being added during cleaning
757      * by other threads participating in lazy migration. The caller must call
758      * TrackedFileSummary.setAllowFlush(true) when cleaning is complete.
759      * This method performs eviction and is not synchronized.
760      * @param logUpdate if true, log any updates to the utilization profile. If
761      * false, only retrieve the new information.
762      */
763     TrackedFileSummary getObsoleteDetail(Long fileNum,
764                                          PackedOffsets packedOffsets,
765                                          boolean logUpdate)
766         throws DatabaseException {
767
768         /* Return if no detail is being tracked. */
769         if (!env.getCleaner().trackDetail) {
770             return null;
771         }
772
773         assert cachePopulated;
774
775         long fileNumVal = fileNum.longValue();
776         List list = new ArrayList ();
777
778         /*
779          * Get an unflushable summary that will remain valid for the duration
780          * of file cleaning.
781          */
782         TrackedFileSummary tfs =
783             env.getLogManager().getUnflushableTrackedSummary(fileNumVal);
784
785         /* Read the summary db. */
786         Locker locker = null;
787         CursorImpl cursor = null;
788         try {
789             locker = new BasicLocker(env);
790             cursor = new CursorImpl(fileSummaryDb, locker);
791             /* Perform eviction in unsynchronized methods. */
792             cursor.setAllowEviction(true);
793
794             DatabaseEntry keyEntry = new DatabaseEntry();
795             DatabaseEntry dataEntry = new DatabaseEntry();
796
797             /* Search by file number. */
798             OperationStatus status = OperationStatus.SUCCESS;
799             if (!getFirstFSLN
800                 (cursor, fileNumVal, keyEntry, dataEntry, LockType.NONE)) {
801                 status = OperationStatus.NOTFOUND;
802             }
803
804             /* Read all LNs for this file number. */
805             while (status == OperationStatus.SUCCESS) {
806
807                 /* Perform eviction once per operation. */
808                 env.getEvictor().doCriticalEviction(true); // backgroundIO
809
810                 FileSummaryLN ln = (FileSummaryLN)
811                     cursor.getCurrentLN(LockType.NONE);
812                 if (ln != null) {
813                     /* Stop if the file number changes. */
814                     if (fileNumVal != ln.getFileNumber(keyEntry.getData())) {
815                         break;
816                     }
817
818                     PackedOffsets offsets = ln.getObsoleteOffsets();
819                     if (offsets != null) {
820                         list.add(offsets.toArray());
821                     }
822
823                     /* Always evict after using a file summary LN. */
824                     cursor.evict();
825                 }
826
827                 status = cursor.getNext
828                     (keyEntry, dataEntry, LockType.NONE,
829                      true, // forward
830 false); // alreadyLatched
831 }
832         } finally {
833             if (cursor != null) {
834                 cursor.releaseBINs();
835                 cursor.close();
836             }
837             if (locker != null) {
838                 locker.operationEnd();
839             }
840         }
841
842         /*
843          * Write out tracked detail, if any, and add its offsets to the list.
844          */
845         if (!tfs.isEmpty()) {
846             PackedOffsets offsets = null;
847             if (logUpdate) {
848                 offsets = putFileSummary(tfs);
849                 if (offsets != null) {
850                     list.add(offsets.toArray());
851                 }
852             } else {
853                 long [] offsetList = tfs.getObsoleteOffsets();
854                 if (offsetList != null) {
855                     list.add(offsetList);
856                 }
857             }
858         }
859
860         /* Merge all offsets into a single array and pack the result. */
861         int size = 0;
862         for (int i = 0; i < list.size(); i += 1) {
863             long[] a = (long[]) list.get(i);
864             size += a.length;
865         }
866         long[] offsets = new long[size];
867         int index = 0;
868         for (int i = 0; i < list.size(); i += 1) {
869             long[] a = (long[]) list.get(i);
870             System.arraycopy(a, 0, offsets, index, a.length);
871             index += a.length;
872         }
873         assert index == offsets.length;
874
875         packedOffsets.pack(offsets);
876
877         return tfs;
878     }
879
880     /**
881      * Populate the profile for file selection. This method performs eviction
882      * and is not synchronized. It must be called before recovery is complete
883      * so that synchronization is unnecessary. It must be called before the
884      * recovery checkpoint so that the checkpoint can flush file summary
885      * information.
886      */
887     public boolean populateCache()
888         throws DatabaseException {
889
890         assert !cachePopulated;
891
892         /* Open the file summary db on first use. */
893         if (!openFileSummaryDatabase()) {
894             /* Db does not exist and this is a read-only environment. */
895             return false;
896         }
897
898         int oldMemorySize = fileSummaryMap.size() *
899             MemoryBudget.UTILIZATION_PROFILE_ENTRY;
900
901         /*
902          * It is possible to have an undeleted FileSummaryLN in the database
903          * for a deleted log file if we crash after deleting a file but before
904          * deleting the FileSummaryLN. Iterate through all FileSummaryLNs and
905          * add them to the cache if their corresponding log file exists. But
906          * delete those records that have no corresponding log file.
907          */
908         Long [] existingFiles = env.getFileManager().getAllFileNumbers();
909         Locker locker = null;
910         CursorImpl cursor = null;
911         try {
912             locker = new BasicLocker(env);
913             cursor = new CursorImpl(fileSummaryDb, locker);
914             /* Perform eviction in unsynchronized methods. */
915             cursor.setAllowEviction(true);
916
917             DatabaseEntry keyEntry = new DatabaseEntry();
918             DatabaseEntry dataEntry = new DatabaseEntry();
919
920             if (cursor.positionFirstOrLast(true, null)) {
921
922                 /* Retrieve the first record. */
923                 OperationStatus status =
924                     cursor.getCurrentAlreadyLatched(keyEntry, dataEntry,
925                                                     LockType.NONE, true);
926                 if (status != OperationStatus.SUCCESS) {
927                     /* The record we're pointing at may be deleted. */
928                     status = cursor.getNext(keyEntry, dataEntry, LockType.NONE,
929                                             true, // go forward
930 false); // do need to latch
931 }
932
933                 while (status == OperationStatus.SUCCESS) {
934
935                     /*
936                      * Perform eviction once per operation. Pass false for
937                      * backgroundIO because this is done during recovery and
938                      * there is no reason to sleep.
939                      */
940                     env.getEvictor().doCriticalEviction(false); // backgroundIO
941
942                     FileSummaryLN ln = (FileSummaryLN)
943                         cursor.getCurrentLN(LockType.NONE);
944
945                     if (ln == null) {
946                         /* Advance past a cleaned record. */
947                         status = cursor.getNext
948                             (keyEntry, dataEntry, LockType.NONE,
949                              true, // go forward
950 false); // do need to latch
951 continue;
952                     }
953
954                     byte[] keyBytes = keyEntry.getData();
955                     boolean isOldVersion = ln.hasStringKey(keyBytes);
956                     long fileNum = ln.getFileNumber(keyBytes);
957                     Long fileNumLong = new Long (fileNum);
958
959                     if (Arrays.binarySearch(existingFiles, fileNumLong) >= 0) {
960
961                         /* File exists, cache the FileSummaryLN. */
962                         fileSummaryMap.put(fileNumLong, ln.getBaseSummary());
963
964                         /*
965                          * Update old version records to the new version. A
966                          * zero sequence number is used to distinguish the
967                          * converted records and to ensure that later records
968                          * will have a greater sequence number.
969                          */
970                         if (isOldVersion) {
971                             insertFileSummary(ln, fileNum, 0);
972                             cursor.latchBIN();
973                             cursor.delete();
974                             cursor.releaseBIN();
975                         } else {
976                             /* Always evict after using a file summary LN. */
977                             cursor.evict();
978                         }
979                     } else {
980
981                         /*
982                          * File does not exist, remove the summary from the map
983                          * and delete all FileSummaryLN records.
984                          */
985                         fileSummaryMap.remove(fileNumLong);
986
987                         if (isOldVersion) {
988                             cursor.latchBIN();
989                             cursor.delete();
990                             cursor.releaseBIN();
991                         } else {
992                             deleteFileSummary(fileNumLong);
993                         }
994
995                         /*
996                          * Do not evict after deleting since the compressor
997                          * would have to fetch it again.
998                          */
999                     }
1000
1001                    /* Go on to the next entry. */
1002                    if (isOldVersion) {
1003
1004                        /* Advance past the single old version record. */
1005                        status = cursor.getNext
1006                            (keyEntry, dataEntry, LockType.NONE,
1007                             true, // go forward
1008 false); // do need to latch
1009 } else {
1010
1011                        /*
1012                         * Skip over other records for this file by adding one
1013                         * to the file number and doing a range search.
1014                         */
1015                        if (!getFirstFSLN
1016                            (cursor,
1017                             fileNum + 1,
1018                             keyEntry, dataEntry,
1019                             LockType.NONE)) {
1020                            status = OperationStatus.NOTFOUND;
1021                        }
1022                    }
1023                }
1024            }
1025        } finally {
1026            if (cursor != null) {
1027                cursor.releaseBINs();
1028                cursor.close();
1029            }
1030            if (locker != null) {
1031                locker.operationEnd();
1032            }
1033
1034            int newMemorySize = fileSummaryMap.size() *
1035                MemoryBudget.UTILIZATION_PROFILE_ENTRY;
1036            MemoryBudget mb = env.getMemoryBudget();
1037            mb.updateMiscMemoryUsage(newMemorySize - oldMemorySize);
1038        }
1039
1040        cachePopulated = true;
1041        return true;
1042    }
1043
1044    /**
1045     * Positions at the most recent LN for the given file number.
1046     */
1047    private boolean getFirstFSLN(CursorImpl cursor,
1048                                 long fileNum,
1049                                 DatabaseEntry keyEntry,
1050                                 DatabaseEntry dataEntry,
1051                                 LockType lockType)
1052        throws DatabaseException {
1053
1054        byte[] keyBytes = FileSummaryLN.makePartialKey(fileNum);
1055        keyEntry.setData(keyBytes);
1056
1057        int result = cursor.searchAndPosition(keyEntry,
1058                                              dataEntry,
1059                                              SearchMode.SET_RANGE,
1060                                              lockType);
1061        if ((result & CursorImpl.FOUND) == 0) {
1062            return false;
1063        }
1064
1065        boolean exactKeyMatch = ((result & CursorImpl.EXACT_KEY) != 0);
1066
1067        if (exactKeyMatch &&
1068            cursor.getCurrentAlreadyLatched
1069                 (keyEntry, dataEntry, lockType, true) !=
1070                    OperationStatus.KEYEMPTY) {
1071            return true;
1072        }
1073
1074        /* Always evict after using a file summary LN. */
1075        cursor.evict(!exactKeyMatch); // alreadyLatched
1076
1077        OperationStatus status = cursor.getNext
1078            (keyEntry, dataEntry, lockType,
1079             true, // forward
1080 !exactKeyMatch); // alreadyLatched
1081
1082        return status == OperationStatus.SUCCESS;
1083    }
1084
1085    /**
1086     * If the file summary db is already open, return, otherwise attempt to
1087     * open it. If the environment is read-only and the database doesn't
1088     * exist, return false. If the environment is read-write the database will
1089     * be created if it doesn't exist.
1090     */
1091    private boolean openFileSummaryDatabase()
1092        throws DatabaseException {
1093
1094        if (fileSummaryDb != null) {
1095            return true;
1096        }
1097        DbTree dbTree = env.getDbMapTree();
1098        Locker autoTxn = null;
1099        boolean operationOk = false;
1100        try {
1101            autoTxn = new AutoTxn(env, new TransactionConfig());
1102            DatabaseImpl db = dbTree.getDb
1103                (autoTxn, DbTree.UTILIZATION_DB_NAME, null);
1104            if (db == null) {
1105                if (env.isReadOnly()) {
1106                    return false;
1107                }
1108                db = dbTree.createDb
1109                    (autoTxn, DbTree.UTILIZATION_DB_NAME,
1110                     new DatabaseConfig(), null);
1111            }
1112            fileSummaryDb = db;
1113            operationOk = true;
1114            return true;
1115        } finally {
1116            if (autoTxn != null) {
1117                autoTxn.operationEnd(operationOk);
1118            }
1119        }
1120    }
1121
1122    /**
1123     * Insert the given LN with the given key values. This method is
1124     * synchronized and may not perform eviction.
1125     */
1126    private synchronized void insertFileSummary(FileSummaryLN ln,
1127                                                long fileNum,
1128                                                int sequence)
1129        throws DatabaseException {
1130
1131        byte[] keyBytes = FileSummaryLN.makeFullKey(fileNum, sequence);
1132
1133        Locker locker = null;
1134        CursorImpl cursor = null;
1135        try {
1136            locker = new BasicLocker(env);
1137            cursor = new CursorImpl(fileSummaryDb, locker);
1138
1139            /* Insert the LN. */
1140            OperationStatus status = cursor.putLN(keyBytes, ln, false);
1141            if (status == OperationStatus.KEYEXIST) {
1142                env.getLogger().log
1143                    (Level.SEVERE,
1144                     "Cleaner duplicate key sequence file=0x" +
1145                     Long.toHexString(fileNum) + " sequence=0x" +
1146                     Long.toHexString(sequence));
1147            }
1148
1149            /* Always evict after using a file summary LN. */
1150            cursor.evict();
1151        } finally {
1152            if (cursor != null) {
1153                cursor.close();
1154            }
1155            if (locker != null) {
1156                locker.operationEnd();
1157            }
1158        }
1159    }
1160
1161    /**
1162     * Checks that all FSLN offsets are indeed obsolete. Assumes that the
1163     * system is quiesent (does not lock LNs). This method is not synchronized
1164     * (because it doesn't access fileSummaryMap) and eviction is allowed.
1165     *
1166     * @return true if no verification failures.
1167     */
1168    public boolean verifyFileSummaryDatabase()
1169        throws DatabaseException {
1170
1171        DatabaseEntry key = new DatabaseEntry();
1172        DatabaseEntry data = new DatabaseEntry();
1173
1174        openFileSummaryDatabase();
1175        Locker locker = null;
1176        CursorImpl cursor = null;
1177        boolean ok = true;
1178        
1179        try {
1180            locker = new BasicLocker(env);
1181            cursor = new CursorImpl(fileSummaryDb, locker);
1182            cursor.setAllowEviction(true);
1183
1184            if (cursor.positionFirstOrLast(true, null)) {
1185
1186                OperationStatus status = cursor.getCurrentAlreadyLatched
1187                    (key, data, LockType.NONE, true);
1188
1189                /* Iterate over all file summary lns. */
1190                while (status == OperationStatus.SUCCESS) {
1191
1192                    /* Perform eviction once per operation. */
1193                    env.getEvictor().doCriticalEviction(true); // backgroundIO
1194
1195                    FileSummaryLN ln = (FileSummaryLN)
1196                        cursor.getCurrentLN(LockType.NONE);
1197
1198                    if (ln != null) {
1199                        long fileNumVal = ln.getFileNumber(key.getData());
1200                        PackedOffsets offsets = ln.getObsoleteOffsets();
1201
1202                        /*
1203                         * Check every offset in the fsln to make sure it's
1204                         * truely obsolete.
1205                         */
1206                        if (offsets != null) {
1207                            long[] vals = offsets.toArray();
1208                            for (int i = 0; i < vals.length; i++) {
1209                                long lsn = DbLsn.makeLsn(fileNumVal, vals[i]);
1210                                if (!verifyLsnIsObsolete(lsn)) {
1211                                    ok = false;
1212                                }
1213                            }
1214                        }
1215
1216                        cursor.evict();
1217                        status = cursor.getNext(key, data, LockType.NONE,
1218                                                true, // forward
1219 false); // already latched
1220 }
1221                }
1222            }
1223        } finally {
1224            if (cursor != null) {
1225                cursor.close();
1226            }
1227            if (locker != null) {
1228                locker.operationEnd();
1229            }
1230        }
1231
1232        return ok;
1233    }
1234
1235    /*
1236     * Return true if the LN at this lsn is obsolete.
1237     */
1238    private boolean verifyLsnIsObsolete(long lsn)
1239        throws DatabaseException {
1240
1241        /* Read the whole entry out of the log. */
1242        Object o = env.getLogManager().getLogEntry(lsn);
1243        if (!(o instanceof LNLogEntry)) {
1244            return true;
1245        }
1246        LNLogEntry entry = (LNLogEntry)o;
1247
1248        /* All deleted LNs are obsolete. */
1249        if (entry.getLN().isDeleted()) {
1250            return true;
1251        }
1252        
1253        /* Find the owning database. */
1254        DatabaseId dbId = entry.getDbId();
1255        DatabaseImpl db = env.getDbMapTree().getDb(dbId);
1256
1257        /*
1258         * The whole database is gone, so this LN is obsolete. No need
1259         * to worry about delete cleanup; this is just verification and
1260         * no cleaning is done.
1261         */
1262        if (db == null || db.isDeleted()) {
1263            return true;
1264        }
1265
1266        /*
1267         * Search down to the bottom most level for the parent of this LN.
1268         */
1269        BIN bin = null;
1270        try {
1271            Tree tree = db.getTree();
1272            TreeLocation location = new TreeLocation();
1273            boolean parentFound = tree.getParentBINForChildLN
1274                (location,
1275                 entry.getKey(),
1276                 entry.getDupKey(),
1277                 entry.getLN(),
1278                 false, // splitsAllowed
1279 true, // findDeletedEntries
1280 false, // searchDupTree ???
1281 false); // updateGeneration
1282 bin = location.bin;
1283            int index = location.index;
1284
1285            /* Is bin latched ? */
1286            if (!parentFound) {
1287                return true;
1288            }
1289
1290            /*
1291             * Now we're at the parent for this LN, whether BIN, DBIN or DIN.
1292             * If knownDeleted, LN is deleted and can be purged.
1293             */
1294            if (bin.isEntryKnownDeleted(index)) {
1295                return true;
1296            }
1297
1298            if (bin.getLsn(index) != lsn) {
1299                return true;
1300            }
1301
1302            /* Oh no -- this lsn is in the tree. */
1303            /* should print, or trace? */
1304            System.err.println("lsn " + DbLsn.getNoFormatString(lsn)+
1305                               " was found in tree.");
1306            return false;
1307        } finally {
1308            if (bin != null) {
1309                bin.releaseLatch();
1310            }
1311        }
1312    }
1313}
1314

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags