Java > Open Source Codes > com > sleepycat > je > log > FileManager


1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002,2006 Oracle. All rights reserved.
5  *
6  * $Id: FileManager.java,v 1.161 2006/11/27 23:07:12 mark Exp $
7  */
8
9 package com.sleepycat.je.log;
10
11 import java.io.File ;
12 import java.io.FileNotFoundException ;
13 import java.io.IOException ;
14 import java.io.RandomAccessFile ;
15 import java.nio.ByteBuffer ;
16 import java.nio.channels.ClosedChannelException ;
17 import java.nio.channels.FileChannel ;
18 import java.nio.channels.FileLock ;
19 import java.nio.channels.OverlappingFileLockException ;
20 import java.util.Arrays ;
21 import java.util.HashMap ;
22 import java.util.Hashtable ;
23 import java.util.Iterator ;
24 import java.util.LinkedList ;
25 import java.util.Map ;
26 import java.util.Random ;
27 import java.util.Set ;
28 import java.util.zip.Checksum ;
29
30 import com.sleepycat.je.DatabaseException;
31 import com.sleepycat.je.EnvironmentStats;
32 import com.sleepycat.je.RunRecoveryException;
33 import com.sleepycat.je.StatsConfig;
34 import com.sleepycat.je.config.EnvironmentParams;
35 import com.sleepycat.je.dbi.DbConfigManager;
36 import com.sleepycat.je.dbi.EnvironmentImpl;
37 import com.sleepycat.je.latch.Latch;
38 import com.sleepycat.je.latch.LatchSupport;
39 import com.sleepycat.je.log.entry.LogEntry;
40 import com.sleepycat.je.utilint.Adler32;
41 import com.sleepycat.je.utilint.DbLsn;
42 import com.sleepycat.je.utilint.HexFormatter;
43
44 /**
45  * The FileManager presents the abstraction of one contiguous file. It doles
46  * out LSNs.
47  */
48 public class FileManager {
49
50     public static class FileMode {
51         public static final FileMode READ_MODE = new FileMode("r");
52         public static final FileMode READWRITE_MODE = new FileMode("rw");
53
54         private String fileModeValue;
55
56         private FileMode(String fileModeValue) {
57             this.fileModeValue = fileModeValue;
58         }
59
60     public String getModeValue() {
61         return fileModeValue;
62     }
63     }
64
65     static boolean IO_EXCEPTION_TESTING = false;
66     private static final String DEBUG_NAME = FileManager.class.getName();
67     /* The number of writes that have been performed. */
68     private static long writeCount = 0;
69     private static long stopOnWriteCount = Long.MAX_VALUE;
70
71     public static final String JE_SUFFIX = ".jdb"; // regular log files
72 public static final String DEL_SUFFIX = ".del"; // cleaned files
73 public static final String BAD_SUFFIX = ".bad"; // corrupt files
74 private static final String LOCK_FILE = "je.lck";// lock file
75 static final String [] DEL_SUFFIXES = { DEL_SUFFIX };
76     static final String [] JE_SUFFIXES = { JE_SUFFIX };
77     private static final String [] JE_AND_DEL_SUFFIXES =
78     { JE_SUFFIX, DEL_SUFFIX };
79
80     /* May be set to false to speed unit tests. */
81     private boolean syncAtFileEnd = true;
82
83     private EnvironmentImpl envImpl;
84     private long maxFileSize;
85     private File dbEnvHome;
86
87     /* True if .del files should be included in the list of log files. */
88     private boolean includeDeletedFiles = false;
89
90     /* File cache */
91     private FileCache fileCache;
92     private Latch fileCacheLatch;
93
94     /* The channel and lock for the je.lck file. */
95     private RandomAccessFile lockFile;
96     private FileChannel channel;
97     private FileLock envLock;
98     private FileLock exclLock;
99
100     /* True if all files should be opened readonly. */
101     private boolean readOnly;
102
103     /* Handles onto log position */
104     private long currentFileNum; // number of the current file
105 private long nextAvailableLsn; // nextLSN is the next one available
106 private long lastUsedLsn; // last LSN used in the current log file
107 private long prevOffset; // Offset to use for the previous pointer
108 private boolean forceNewFile; // Force new file on next write
109
110     /*
111      * Saved versions of above. Save this in case a write causes an
112      * IOException, we can back the log up to the last known good LSN.
113      */
114     private long savedCurrentFileNum;
115     private long savedNextAvailableLsn; // nextLSN is the next one available
116 private long savedLastUsedLsn; // last LSN used in the current log file
117 private long savedPrevOffset; // Offset to use for the previous pointer
118 private boolean savedForceNewFile;
119
120     /* endOfLog is used for writes and fsyncs to the end of the log. */
121     private LogEndFileDescriptor endOfLog;
122
123     /* group commit sync */
124     private FSyncManager syncManager;
125
126     /*
127      * When we bump the LSNs over to a new file, we must remember the last LSN
128      * of the previous file so we can set the prevOffset field of the file
129      * header appropriately. We have to save it in a map because there's a time
130      * lag between when we know what the last LSN is and when we actually do
131      * the file write, because LSN bumping is done before we get a write
132      * buffer. This map is keyed by file num->last LSN.
133      */
134     private Map perFileLastUsedLsn;
135
136     /* Whether to use NIO for file I/O. */
137     private boolean useNIO;
138     
139     /*
140      * If non-0, do NIO in chunks of this size.
141      */
142     private long chunkedNIOSize = 0;
143
144     /**
145      * Set up the file cache and initialize the file manager to point to the
146      * beginning of the log.
147      *
148      * @param configManager
149      * @param dbEnvHome environment home directory
150      */
151     public FileManager(EnvironmentImpl envImpl,
152                        File dbEnvHome,
153                        boolean readOnly)
154         throws DatabaseException {
155
156         this.envImpl = envImpl;
157         this.dbEnvHome = dbEnvHome;
158         this.readOnly = readOnly;
159
160         /* Read configurations. */
161         DbConfigManager configManager = envImpl.getConfigManager();
162         maxFileSize = configManager.getLong(EnvironmentParams.LOG_FILE_MAX);
163
164         useNIO =
165         configManager.getBoolean(EnvironmentParams.LOG_USE_NIO);
166         chunkedNIOSize =
167         configManager.getLong(EnvironmentParams.LOG_CHUNKED_NIO);
168         boolean directNIO =
169             configManager.getBoolean(EnvironmentParams.LOG_DIRECT_NIO);
170
171         if (!useNIO && (chunkedNIOSize > 0 || directNIO)) {
172             throw new IllegalArgumentException
173                 (EnvironmentParams.LOG_USE_NIO.getName() +
174                  " is false and therefore " +
175                  EnvironmentParams.LOG_DIRECT_NIO.getName() +
176                  " or " +
177                  EnvironmentParams.LOG_CHUNKED_NIO.getName() +
178                  " may not be used.");
179         }
180
181         if (!envImpl.isMemOnly()) {
182             if (!dbEnvHome.exists()) {
183                 throw new LogException("Environment home " + dbEnvHome +
184                                          " doesn't exist");
185             }
186             lockEnvironment(readOnly, false);
187         }
188
189         /* Cache of files. */
190         fileCache = new FileCache(configManager);
191         fileCacheLatch =
192         LatchSupport.makeLatch(DEBUG_NAME + "_fileCache", envImpl);
193
194         /* Start out as if no log existed. */
195         currentFileNum = 0L;
196         nextAvailableLsn = DbLsn.makeLsn(currentFileNum,
197                      firstLogEntryOffset());
198         lastUsedLsn = DbLsn.NULL_LSN;
199         perFileLastUsedLsn = new HashMap ();
200         prevOffset = 0L;
201         endOfLog = new LogEndFileDescriptor();
202         forceNewFile = false;
203     saveLastPosition();
204
205         String stopOnWriteProp = System.getProperty("je.debug.stopOnWrite");
206         if (stopOnWriteProp != null) {
207             stopOnWriteCount = Long.parseLong(stopOnWriteProp);
208         }
209
210         syncManager = new FSyncManager(envImpl);
211     }
212
213     /**
214      * Set the file manager's "end of log".
215      *
216      * @param nextAvailableLsn LSN to be used for the next log entry
217      * @param lastUsedLsn last LSN to have a valid entry, may be null
218      * @param prevOffset value to use for the prevOffset of the next entry.
219      * If the beginning of the file, this is 0.
220      */
221     public void setLastPosition(long nextAvailableLsn,
222                                 long lastUsedLsn,
223                                 long prevOffset) {
224         this.lastUsedLsn = lastUsedLsn;
225         perFileLastUsedLsn.put(new Long (DbLsn.getFileNumber(lastUsedLsn)),
226                                new Long (lastUsedLsn));
227         this.nextAvailableLsn = nextAvailableLsn;
228         currentFileNum = DbLsn.getFileNumber(this.nextAvailableLsn);
229         this.prevOffset = prevOffset;
230     saveLastPosition();
231     }
232
233     /*
234      * Cause the current LSN state to be saved in case we fail after we have
235      * bumped the lsn pointer but before we've successfully marshalled into the
236      * log buffer.
237      */
238     void saveLastPosition() {
239     savedNextAvailableLsn = nextAvailableLsn;
240     savedLastUsedLsn = lastUsedLsn;
241     savedPrevOffset = prevOffset;
242         savedForceNewFile = forceNewFile;
243         savedCurrentFileNum = currentFileNum;
244     }
245
246     void restoreLastPosition() {
247     nextAvailableLsn = savedNextAvailableLsn;
248     lastUsedLsn = savedLastUsedLsn;
249     prevOffset = savedPrevOffset;
250         forceNewFile = savedForceNewFile;
251         currentFileNum = savedCurrentFileNum;
252     }
253
254     /**
255      * May be used to disable sync at file end to speed unit tests.
256      * Must only be used for unit testing, since log corruption may result.
257      */
258     public void setSyncAtFileEnd(boolean sync) {
259         syncAtFileEnd = sync;
260     }
261
262     /*
263      * File management
264      */
265
266     /**
267      * public for cleaner.
268      *
269      * @return the number of the first file in this environment.
270      */
271     public Long getFirstFileNum() {
272         return getFileNum(true);
273     }
274
275     public boolean getReadOnly() {
276     return readOnly;
277     }
278
279     /**
280      * @return the number of the last file in this environment.
281      */
282     public Long getLastFileNum() {
283         return getFileNum(false);
284     }
285
286     /*
287      * For unit tests.
288      */
289     public long getCurrentFileNum() {
290     return currentFileNum;
291     }
292
293     public void setIncludeDeletedFiles(boolean includeDeletedFiles) {
294     this.includeDeletedFiles = includeDeletedFiles;
295     }
296
297     /**
298      * Get all JE file numbers.
299      * @return an array of all JE file numbers.
300      */
301     public Long [] getAllFileNumbers() {
302         /* Get all the names in sorted order. */
303         String [] names = listFiles(JE_SUFFIXES);
304         Long [] nums = new Long [names.length];
305         for (int i = 0; i < nums.length; i += 1) {
306             nums[i] = getNumFromName(names[i]);
307         }
308         return nums;
309     }
310
311     /**
312      * Get the next file number before/after currentFileNum.
313      * @param currentFileNum the file we're at right now. Note that
314      * it may not exist, if it's been cleaned and renamed.
315      * @param forward if true, we want the next larger file, if false
316      * we want the previous file
317      * @return null if there is no following file, or if filenum doesn't exist
318      */
319     public Long getFollowingFileNum(long currentFileNum, boolean forward) {
320         /* Get all the names in sorted order. */
321         String [] names = listFiles(JE_SUFFIXES);
322
323         /* Search for the current file. */
324         String searchName = getFileName(currentFileNum, JE_SUFFIX);
325         int foundIdx = Arrays.binarySearch(names, searchName);
326
327         boolean foundTarget = false;
328         if (foundIdx >= 0) {
329             if (forward) {
330                 foundIdx++;
331             } else {
332                 foundIdx --;
333             }
334         } else {
335
336             /*
337              * currentFileNum not found (might have been cleaned). FoundIdx
338              * will be (-insertionPoint - 1).
339              */
340             foundIdx = Math.abs(foundIdx + 1);
341             if (!forward) {
342                 foundIdx--;
343             }
344         }
345
346         /* The current fileNum is found, return the next or prev file. */
347         if (forward && (foundIdx < names.length)) {
348             foundTarget = true;
349         } else if (!forward && (foundIdx > -1)) {
350             foundTarget = true;
351         }
352
353         if (foundTarget) {
354             return getNumFromName(names[foundIdx]);
355         } else {
356             return null;
357         }
358     }
359
360     /**
361      * @return true if there are any files at all.
362      */
363     public boolean filesExist() {
364         String [] names = listFiles(JE_SUFFIXES);
365         return (names.length != 0);
366     }
367
368     /**
369      * Get the first or last file number in the set of je files.
370      *
371      * @param first if true, get the first file, else get the last file
372      * @return the file number or null if no files exist
373      */
374     private Long getFileNum(boolean first) {
375         String [] names = listFiles(JE_SUFFIXES);
376         if (names.length == 0) {
377             return null;
378         } else {
379             int index = 0;
380             if (!first) {
381                 index = names.length - 1;
382             }
383             return getNumFromName(names[index]);
384         }
385     }
386
387     /**
388      * Get the file number from a file name.
389      *
390      * @param the file name
391      * @return the file number
392      */
393     public Long getNumFromName(String fileName) {
394         String fileNumber = fileName.substring(0, fileName.indexOf("."));
395         return new Long (Long.parseLong(fileNumber, 16));
396     }
397
398     /**
399      * Find je files. Return names sorted in ascending fashion.
400      * @param suffix which type of file we're looking for
401      * @return array of file names
402      */
403     public String [] listFiles(String [] suffixes) {
404         String [] fileNames = dbEnvHome.list(new JEFileFilter(suffixes));
405         if (fileNames != null) {
406             Arrays.sort(fileNames);
407         } else {
408             fileNames = new String [0];
409         }
410         return fileNames;
411     }
412
413     /**
414      * Find .jdb files which are >= the minimimum file number and
415      * <= the maximum file number.
416      * Return names sorted in ascending fashion.
417      *
418      * @return array of file names
419      */
420     public String [] listFiles(long minFileNumber, long maxFileNumber) {
421
422         String [] fileNames = dbEnvHome.list(new JEFileFilter(JE_SUFFIXES,
423                                                              minFileNumber,
424                                                              maxFileNumber));
425         Arrays.sort(fileNames);
426         return fileNames;
427     }
428
429    /**
430      * Find je files, flavor for unit test support.
431      *
432      * @param suffix which type of file we're looking for
433      * @return array of file names
434      */
435     public static String [] listFiles(File envDirFile, String [] suffixes) {
436         String [] fileNames = envDirFile.list(new JEFileFilter(suffixes));
437         if (fileNames != null) {
438             Arrays.sort(fileNames);
439         } else {
440             fileNames = new String [0];
441         }
442         return fileNames;
443     }
444
445     /**
446      * @return the full file name and path for the nth je file.
447      */
448     String [] getFullFileNames(long fileNum) {
449     if (includeDeletedFiles) {
450         int nSuffixes = JE_AND_DEL_SUFFIXES.length;
451         String [] ret = new String [nSuffixes];
452         for (int i = 0; i < nSuffixes; i++) {
453         ret[i] = getFullFileName(getFileName(fileNum,
454                                                      JE_AND_DEL_SUFFIXES[i]));
455         }
456         return ret;
457     } else {
458         return new String []
459                 { getFullFileName(getFileName(fileNum, JE_SUFFIX)) };
460     }
461     }
462
463     /**
464      * @return the full file name and path for the given file number and
465      * suffix.
466      */
467     public String getFullFileName(long fileNum, String suffix) {
468         return getFullFileName(getFileName(fileNum, suffix));
469     }
470
471     /**
472      * @return the full file name and path for this file name.
473      */
474     private String getFullFileName(String fileName) {
475         return dbEnvHome + File.separator + fileName;
476     }
477
478     /**
479      * @return the file name for the nth file.
480      */
481     public static String getFileName(long fileNum, String suffix) {
482
483         /*
484          * HexFormatter generates a 0 padded string starting with 0x. We want
485          * the right most 8 digits, so start at 10.
486          */
487         return (HexFormatter.formatLong(fileNum).substring(10) + suffix);
488     }
489
490     /**
491      * Rename this file to NNNNNNNN.suffix. If that file already exists, try
492      * NNNNNNNN.suffix.1, etc. Used for deleting files or moving corrupt files
493      * aside.
494      *
495      * @param fileNum the file we want to move
496      * @param newSuffix the new file suffix
497      */
498     public void renameFile(long fileNum, String newSuffix)
499         throws DatabaseException, IOException {
500
501         int repeatNum = 0;
502         boolean renamed = false;
503         while (!renamed) {
504             String generation = "";
505             if (repeatNum > 0) {
506                 generation = "." + repeatNum;
507             }
508             String newName =
509         getFullFileName(getFileName(fileNum, newSuffix) + generation);
510             File targetFile = new File (newName);
511             if (targetFile.exists()) {
512                 repeatNum++;
513             } else {
514                 String oldFileName = getFullFileNames(fileNum)[0];
515                 clearFileCache(fileNum);
516                 File oldFile = new File (oldFileName);
517                 if (oldFile.renameTo(targetFile)) {
518                     renamed = true;
519                 } else {
520                     throw new LogException("Couldn't rename " + oldFileName +
521                                              " to " + newName);
522                 }
523             }
524         }
525     }
526
527     /**
528      * Delete log file NNNNNNNN.
529      *
530      * @param fileNum the file we want to move
531      */
532     public void deleteFile(long fileNum)
533         throws DatabaseException, IOException {
534
535     String fileName = getFullFileNames(fileNum)[0];
536     clearFileCache(fileNum);
537     File file = new File (fileName);
538     boolean done = file.delete();
539     if (!done) {
540         throw new LogException
541         ("Couldn't delete " + file);
542     }
543     }
544
545     /**
546      * Return a read only file handle that corresponds the this file number.
547      * Retrieve it from the cache or open it anew and validate the file header.
548      * This method takes a latch on this file, so that the file descriptor will
549      * be held in the cache as long as it's in use. When the user is done with
550      * the file, the latch must be released.
551      *
552      * @param fileNum which file
553      * @return the file handle for the existing or newly created file
554      */
555     FileHandle getFileHandle(long fileNum)
556         throws LogException, DatabaseException {
557
558         /* Check the file cache for this file. */
559         Long fileId = new Long (fileNum);
560         FileHandle fileHandle = null;
561
562         /**
563          * Loop until we get an open FileHandle.
564          */
565         while (true) {
566
567             /*
568              * The file cache is intentionally not latched here so that it's
569              * not a bottleneck in the fast path. We check that the file
570              * handle that we get back is really still open after we latch it
571              * down below.
572              */
573             fileHandle = fileCache.get(fileId);
574
575             /* The file wasn't in the cache. */
576             if (fileHandle == null) {
577                 fileCacheLatch.acquire();
578                 try {
579                     /* Check the file cache again under the latch. */
580                     fileHandle = fileCache.get(fileId);
581                     if (fileHandle == null) {
582
583                         fileHandle =
584                 makeFileHandle(fileNum, FileMode.READ_MODE);
585
586                         /* Put it into the cache. */
587                         fileCache.add(fileId, fileHandle);
588                     }
589                 } finally {
590                     fileCacheLatch.release();
591                 }
592             }
593
594             /* Get latch before returning */
595             fileHandle.latch();
596
597             /*
598              * We may have obtained this file handle outside the file cache
599              * latch, so we have to test that the handle is still valid. If
600              * it's not, then loop back and try again.
601              */
602             if (fileHandle.getFile() == null) {
603                 fileHandle.release();
604             } else {
605                 break;
606             }
607         }
608
609         return fileHandle;
610     }
611
612     private FileHandle makeFileHandle(long fileNum, FileMode mode)
613         throws DatabaseException {
614
615         String [] fileNames = getFullFileNames(fileNum);
616         RandomAccessFile newFile = null;
617         String fileName = null;
618         try {
619
620             /*
621          * Open the file. Note that we are going to try a few names to open
622          * this file -- we'll try for N.jdb, and if that doesn't exist and
623          * we're configured to look for all types, we'll look for N.del.
624              */
625             FileNotFoundException FNFE = null;
626             for (int i = 0; i < fileNames.length; i++) {
627                 fileName = fileNames[i];
628                 try {
629                     newFile =
630             new RandomAccessFile (fileName, mode.getModeValue());
631                     break;
632                 } catch (FileNotFoundException e) {
633                     /* Save the first exception thrown. */
634                     if (FNFE == null) {
635                         FNFE = e;
636                     }
637                 }
638             }
639
640             /*
641          * If we didn't find the file or couldn't create it, rethrow the
642          * exception.
643          */
644             if (newFile == null) {
645                 throw FNFE;
646             }
647
648             boolean oldHeaderVersion = false;
649
650             if (newFile.length() == 0) {
651                 /*
652                  * If the file is empty, reinitialize it if we can. If
653                  * not, send the file handle back up; the calling code will
654                  * deal with the fact that there's nothing there.
655                  */
656                 if (mode == FileMode.READWRITE_MODE) {
657                     /* An empty file, write a header. */
658                     long lastLsn = DbLsn.longToLsn((Long )
659                                                    perFileLastUsedLsn.remove
660                                                    (new Long (fileNum - 1)));
661                     long headerPrevOffset = 0;
662                     if (lastLsn != DbLsn.NULL_LSN) {
663                         headerPrevOffset = DbLsn.getFileOffset(lastLsn);
664                     }
665                     FileHeader fileHeader =
666                         new FileHeader(fileNum,
667                                        headerPrevOffset);
668                     writeFileHeader(newFile,
669                                     fileName,
670                                     fileHeader);
671                 }
672             } else {
673                 /* A non-empty file, check the header */
674                 oldHeaderVersion =
675                     readAndValidateFileHeader(newFile, fileName, fileNum);
676             }
677             return new FileHandle
678                 (newFile, fileName, envImpl, oldHeaderVersion);
679         } catch (FileNotFoundException e) {
680             throw new LogFileNotFoundException
681                 ("Couldn't open file " + fileName + ": " +
682                  e.getMessage());
683         } catch (DbChecksumException e) {
684
685             /*
686              * Let this exception go as a checksum exception, so it sets the
687              * run recovery state correctly.
688              */
689             closeFileInErrorCase(newFile);
690             throw new DbChecksumException
691                 (envImpl, "Couldn't open file " + fileName, e);
692         } catch (Throwable t) {
693
694             /*
695              * Catch Throwable here (rather than exception) because in unit
696              * test mode, we run assertions and they throw errors. We want to
697              * clean up the file object in all cases.
698              */
699             closeFileInErrorCase(newFile);
700             throw new DatabaseException
701         ("Couldn't open file " + fileName + ": " + t, t);
702         }
703     }
704
705     /**
706      * Close this file and eat any exceptions. Used in catch clauses.
707      */
708     private void closeFileInErrorCase(RandomAccessFile file) {
709         try {
710             if (file != null) {
711                 file.close();
712             }
713         } catch (IOException e) {
714
715         /*
716          * Klockwork - ok
717              * Couldn't close file, oh well.
718          */
719         }
720     }
721
722     /**
723      * Read the given je log file and validate the header.
724      *
725      * @throws DatabaseException if the file header isn't valid
726      *
727      * @return whether the file header has an old version number.
728      */
729     private boolean readAndValidateFileHeader(RandomAccessFile file,
730                                               String fileName,
731                                               long fileNum)
732         throws DatabaseException, IOException {
733
734         /*
735          * Read the file header from this file. It's always the first log
736          * entry.
737          */
738         LogManager logManager = envImpl.getLogManager();
739         LogEntry headerEntry =
740             logManager.getLogEntry(DbLsn.makeLsn(fileNum, 0), file);
741         FileHeader header = (FileHeader) headerEntry.getMainItem();
742         return header.validate(fileName, fileNum);
743     }
744
745     /**
746      * Write a proper file header to the given file.
747      */
748     private void writeFileHeader(RandomAccessFile file,
749                                  String fileName,
750                                  FileHeader header)
751         throws DatabaseException, IOException {
752
753     /*
754      * Fail loudly if the environment is invalid. A RunRecoveryException
755      * must have occurred.
756      */
757     envImpl.checkIfInvalid();
758
759         /*
760          * Fail silent if the environment is not open.
761          */
762         if (envImpl.mayNotWrite()) {
763             return;
764         }
765
766         /* Serialize the header into this buffer. */
767         int headerSize = header.getLogSize();
768         int entrySize = headerSize + LogManager.HEADER_BYTES;
769         ByteBuffer headerBuf = envImpl.getLogManager().
770         putIntoBuffer(header, headerSize, 0, false, entrySize);
771         
772     if (++writeCount >= stopOnWriteCount) {
773         Runtime.getRuntime().halt(0xff);
774     }
775
776         /* Write the buffer into the channel. */
777         int bytesWritten;
778         try {
779             if (RUNRECOVERY_EXCEPTION_TESTING) {
780                 generateRunRecoveryException(file, headerBuf, 0);
781             }
782             bytesWritten = writeToFile(file, headerBuf, 0);
783         } catch (ClosedChannelException e) {
784
785             /*
786              * The channel should never be closed. It may be closed because
787              * of an interrupt received by another thread. See SR [#10463]
788              */
789             throw new RunRecoveryException
790         (envImpl, "Channel closed, may be due to thread interrupt", e);
791         } catch (IOException e) {
792             /* Possibly an out of disk exception. */
793             throw new RunRecoveryException
794         (envImpl, "IOException caught: " + e);
795         }
796
797         if (bytesWritten != entrySize) {
798             throw new LogException
799         ("File " + fileName +
800          " was created with an incomplete header. Only " +
801          bytesWritten + " bytes were written.");
802         }
803     }
804
805     /**
806      * @return the prevOffset field stored in the file header.
807      */
808     long getFileHeaderPrevOffset(long fileNum)
809         throws IOException , DatabaseException {
810         
811         LogEntry headerEntry =
812             envImpl.getLogManager().getLogEntry(DbLsn.makeLsn(fileNum, 0));
813         FileHeader header = (FileHeader) headerEntry.getMainItem();
814         return header.getLastEntryInPrevFileOffset();
815     }
816
817     /*
818      * Support for writing new log entries
819      */
820
821     /**
822      * @return the file offset of the last LSN that was used. For constructing
823      * the headers of log entries. If the last LSN that was used was in a
824      * previous file, or this is the very first LSN of the whole system, return
825      * 0.
826      */
827     long getPrevEntryOffset() {
828         return prevOffset;
829     }
830
831     /**
832      * Increase the current log position by "size" bytes. Move the prevOffset
833      * pointer along.
834      *
835      * @param size is an unsigned int
836      * @return true if we flipped to the next log file.
837      */
838     boolean bumpLsn(long size) {
839
840         /* Save copy of initial lsn state. */
841         saveLastPosition();
842
843         boolean flippedFiles = false;
844
845         if (forceNewFile ||
846             (DbLsn.getFileOffset(nextAvailableLsn) + size) > maxFileSize) {
847
848             forceNewFile = false;
849
850             /* Move to another file. */
851             currentFileNum++;
852
853             /* Remember the last used LSN of the previous file. */
854             if (lastUsedLsn != DbLsn.NULL_LSN) {
855                 perFileLastUsedLsn.put
856             (new Long (DbLsn.getFileNumber(lastUsedLsn)),
857              new Long (lastUsedLsn));
858             }
859             prevOffset = 0;
860             lastUsedLsn = DbLsn.makeLsn(currentFileNum,
861                     firstLogEntryOffset());
862             flippedFiles = true;
863         } else {
864             if (lastUsedLsn == DbLsn.NULL_LSN) {
865                 prevOffset = 0;
866             } else {
867                 prevOffset = DbLsn.getFileOffset(lastUsedLsn);
868             }
869             lastUsedLsn = nextAvailableLsn;
870         }
871         nextAvailableLsn =
872         DbLsn.makeLsn(DbLsn.getFileNumber(lastUsedLsn),
873               (DbLsn.getFileOffset(lastUsedLsn) + size));
874
875         return flippedFiles;
876     }
877
878     /**
879      * Write out a log buffer to the file.
880      * @param fullBuffer buffer to write
881      */
882     void writeLogBuffer(LogBuffer fullBuffer)
883         throws DatabaseException {
884
885     /*
886      * Fail loudly if the environment is invalid. A RunRecoveryException
887      * must have occurred.
888      */
889     envImpl.checkIfInvalid();
890
891         /*
892          * Fail silent if the environment is not open.
893          */
894         if (envImpl.mayNotWrite()) {
895             return;
896         }
897
898         /* Use the LSN to figure out what file to write this buffer to. */
899         long firstLsn = fullBuffer.getFirstLsn();
900
901         /*
902          * Is there anything in this write buffer? We could have been called by
903          * the environment shutdown, and nothing is actually in the buffer.
904          */
905         if (firstLsn != DbLsn.NULL_LSN) {
906             
907             RandomAccessFile file =
908                 endOfLog.getWritableFile(DbLsn.getFileNumber(firstLsn));
909             ByteBuffer data = fullBuffer.getDataBuffer();
910  
911             if (++writeCount >= stopOnWriteCount) {
912                 Runtime.getRuntime().halt(0xff);
913             }
914
915             try {
916
917                 /*
918                  * Check that we do not overwrite unless the file only contains
919                  * a header [#11915] [#12616].
920                  */
921                 assert fullBuffer.getRewriteAllowed() ||
922             (DbLsn.getFileOffset(firstLsn) >= file.length() ||
923                      file.length() == firstLogEntryOffset()) :
924                         "FileManager would overwrite non-empty file 0x" +
925                         Long.toHexString(DbLsn.getFileNumber(firstLsn)) +
926                         " lsnOffset=0x" +
927                         Long.toHexString(DbLsn.getFileOffset(firstLsn)) +
928                         " fileLength=0x" +
929                         Long.toHexString(file.length());
930
931                 if (IO_EXCEPTION_TESTING) {
932             throw new IOException ("generated for testing");
933                 }
934                 if (RUNRECOVERY_EXCEPTION_TESTING) {
935                     generateRunRecoveryException
936                         (file, data, DbLsn.getFileOffset(firstLsn));
937                 }
938         writeToFile(file, data, DbLsn.getFileOffset(firstLsn));
939             } catch (ClosedChannelException e) {
940
941                 /*
942                  * The file should never be closed. It may be closed because
943                  * of an interrupt received by another thread. See SR [#10463].
944                  */
945                 throw new RunRecoveryException
946             (envImpl, "File closed, may be due to thread interrupt",
947              e);
948             } catch (IOException IOE) {
949
950         /*
951          * Possibly an out of disk exception, but java.io will only
952          * tell us IOException with no indication of whether it's out
953          * of disk or something else.
954          *
955          * Since we can't tell what sectors were actually written to
956          * disk, we need to change any commit records that might have
957          * made it out to disk to abort records. If they made it to
958          * disk on the write, then rewriting should allow them to be
959          * rewritten. See [11271].
960          */
961         abortCommittedTxns(data);
962         try {
963             if (IO_EXCEPTION_TESTING) {
964             throw new IOException ("generated for testing");
965             }
966             writeToFile(file, data, DbLsn.getFileOffset(firstLsn));
967         } catch (IOException IOE2) {
968             fullBuffer.setRewriteAllowed();
969             throw new DatabaseException(IOE2);
970         }
971         }
972
973         assert EnvironmentImpl.maybeForceYield();
974         }
975     }
976
977     /**
978      * Write a buffer to a file at a given offset, using NIO if so configured.
979      */
980     private int writeToFile(RandomAccessFile file,
981                             ByteBuffer data,
982                             long destOffset)
983     throws IOException , DatabaseException {
984
985         int totalBytesWritten = 0;
986         if (useNIO) {
987             FileChannel channel = file.getChannel();
988
989             if (chunkedNIOSize > 0) {
990
991                 /*
992                  * We can't change the limit without impacting readers that
993                  * might find this buffer in the buffer pool. Duplicate the
994                  * buffer so we can set the limit independently.
995                  */
996                 ByteBuffer useData = data.duplicate();
997
998                 /*
999                  * Write small chunks of data by manipulating the position and
1000                 * limit properties of the buffer, and submitting it for
1001                 * writing repeatedly.
1002                 *
1003                 * For each chunk, the limit is set to the position +
1004                 * chunkedNIOSize, capped by the original limit of the buffer.
1005                 *
1006                 * Preconditions: data to be written is betweek data.position()
1007                 * and data.limit()
1008
1009                 * Postconditions: data.limit() has not changed,
1010                 * data.position() == data.limit(), offset of the channel has
1011                 * not been modified.
1012                 */
1013                int originalLimit = useData.limit();
1014                useData.limit(useData.position());
1015                while (useData.limit() < originalLimit) {
1016                    useData.limit((int)
1017                                  (Math.min(useData.limit() + chunkedNIOSize,
1018                                            originalLimit)));
1019                    int bytesWritten = channel.write(useData, destOffset);
1020                    destOffset += bytesWritten;
1021                    totalBytesWritten += bytesWritten;
1022                }
1023            } else {
1024
1025                /*
1026                 * Perform a single write using NIO.
1027                 */
1028                totalBytesWritten = channel.write(data, destOffset);
1029            }
1030        } else {
1031
1032            /*
1033             * Perform a RandomAccessFile write and update the buffer position.
1034             * ByteBuffer.array() is safe to use since all non-direct
1035             * ByteBuffers have a backing array. Synchronization on the file
1036             * object is needed because two threads may call seek() on the same
1037             * file object.
1038             */
1039            synchronized (file) {
1040                assert data.hasArray();
1041                assert data.arrayOffset() == 0;
1042
1043                int pos = data.position();
1044                int size = data.limit() - pos;
1045                file.seek(destOffset);
1046                file.write(data.array(), pos, size);
1047                data.position(pos + size);
1048                totalBytesWritten = size;
1049            }
1050        }
1051        return totalBytesWritten;
1052    }
1053 
1054    /**
1055     * Read a buffer from a file at a given offset, using NIO if so configured.
1056     */
1057    void readFromFile(RandomAccessFile file,
1058                      ByteBuffer readBuffer,
1059                      long offset)
1060    throws IOException {
1061 
1062        if (useNIO) {
1063            FileChannel channel = file.getChannel();
1064
1065            if (chunkedNIOSize > 0) {
1066
1067                /*
1068                 * Read a chunk at a time to prevent large direct memory
1069                 * allocations by NIO.
1070                 */
1071                int readLength = readBuffer.limit();
1072                long currentPosition = offset;
1073                while (readBuffer.position() < readLength) {
1074                    readBuffer.limit((int)
1075                                     (Math.min(readBuffer.limit() +
1076                                               chunkedNIOSize,
1077                                               readLength)));
1078                    int bytesRead = channel.read(readBuffer, currentPosition);
1079      
1080                    if (bytesRead < 1)
1081                        break;
1082      
1083                    currentPosition += bytesRead;
1084                }
1085            } else {
1086
1087                /*
1088                 * Perform a single read using NIO.
1089                 */
1090                channel.read(readBuffer, offset);
1091            }
1092        } else {
1093
1094            /*
1095             * Perform a RandomAccessFile read and update the buffer position.
1096             * ByteBuffer.array() is safe to use since all non-direct
1097             * ByteBuffers have a backing array. Synchronization on the file
1098             * object is needed because two threads may call seek() on the same
1099             * file object.
1100             */
1101            synchronized (file) {
1102                assert readBuffer.hasArray();
1103                assert readBuffer.arrayOffset() == 0;
1104
1105                int pos = readBuffer.position();
1106                int size = readBuffer.limit() - pos;
1107                file.seek(offset);
1108                int bytesRead = file.read(readBuffer.array(), pos, size);
1109                if (bytesRead > 0) {
1110                    readBuffer.position(pos + bytesRead);
1111                }
1112            }
1113        }
1114    }
1115
1116    /*
1117     * Iterate through a buffer looking for commit records. Change all commit
1118     * records to abort records.
1119     */
1120    private void abortCommittedTxns(ByteBuffer data) {
1121    final byte commitType = LogEntryType.LOG_TXN_COMMIT.getTypeNum();
1122    final byte abortType = LogEntryType.LOG_TXN_ABORT.getTypeNum();
1123    data.position(0);
1124
1125    while (data.remaining() > 0) {
1126        int recStartPos = data.position();
1127        data.position(recStartPos + LogManager.HEADER_ENTRY_TYPE_OFFSET);
1128        int typePos = data.position();
1129        byte entryType = data.get();
1130        boolean recomputeChecksum = false;
1131        if (entryType == commitType) {
1132        data.position(typePos);
1133        data.put(abortType);
1134        recomputeChecksum = true;
1135        }
1136        /* Move byte buffer past version. */
1137        byte version = data.get();
1138        /* Read the size, skipping over the prev offset. */
1139        data.position(data.position() + LogManager.PREV_BYTES);
1140        int itemSize = LogUtils.readInt(data);
1141        int itemDataStartPos = data.position();
1142        if (recomputeChecksum) {
1143        Checksum checksum = Adler32.makeChecksum();
1144        data.position(recStartPos);
1145        /* Calculate the checksum and write it into the buffer. */
1146        int nChecksumBytes = itemSize +
1147            (LogManager.HEADER_BYTES - LogManager.CHECKSUM_BYTES);
1148        byte[] checksumBytes = new byte[nChecksumBytes];
1149        System.arraycopy(data.array(),
1150                 recStartPos + LogManager.CHECKSUM_BYTES,
1151                 checksumBytes, 0, nChecksumBytes);
1152        checksum.update(checksumBytes, 0, nChecksumBytes);
1153        LogUtils.writeUnsignedInt(data, checksum.getValue());
1154        }
1155        data.position(itemDataStartPos + itemSize);
1156    }
1157    data.position(0);
1158    }
1159
1160    /**
1161     * FSync the end of the log.
1162     */
1163    void syncLogEnd()
1164        throws DatabaseException {
1165
1166        try {
1167            endOfLog.force();
1168        } catch (IOException e) {
1169            throw new DatabaseException(e);
1170        }
1171    }
1172
1173    /**
1174     * Sync the end of the log, close off this log file. Should only be called
1175     * under the log write latch.
1176     */
1177    void syncLogEndAndFinishFile()
1178        throws DatabaseException, IOException {
1179        
1180        if (syncAtFileEnd) {
1181            syncLogEnd();
1182        }
1183        endOfLog.close();
1184    }
1185
1186    /**
1187     * Flush a file using the group sync mechanism, trying to amortize off
1188     * other syncs.
1189     */
1190    void groupSync()
1191        throws DatabaseException {
1192
1193        syncManager.fsync();
1194    }
1195
1196    /**
1197     * Close all file handles and empty the cache.
1198     */
1199    public void clear()
1200        throws IOException , DatabaseException {
1201
1202        fileCacheLatch.acquire();
1203        try {
1204            fileCache.clear();
1205        } finally {
1206            fileCacheLatch.release();
1207        }
1208
1209        endOfLog.close();
1210    }
1211
1212    /**
1213     * Clear the file lock.
1214     */
1215    public void close()
1216        throws IOException , DatabaseException {
1217
1218        if (envLock != null) {
1219            envLock.release();
1220        }
1221
1222        if (exclLock != null) {
1223            exclLock.release();
1224        }
1225
1226        if (channel != null) {
1227            channel.close();
1228        }
1229
1230        if (lockFile != null) {
1231            lockFile.close();
1232            lockFile = null;
1233        }
1234    }
1235
1236    /**
1237     * Lock the environment. Return true if the lock was acquired. If
1238     * exclusive is false, then this implements a single writer, multiple
1239     * reader lock. If exclusive is true, then implement an exclusive lock.
1240     *
1241     * There is a lock file and there are two regions of the lock file: byte 0,
1242     * and byte 1. Byte 0 is the exclusive writer process area of the lock
1243     * file. If an environment is opened for write, then it attempts to take
1244     * an exclusive write lock on byte 0. Byte 1 is the shared reader process
1245     * area of the lock file. If an environment is opened for read-only, then
1246     * it attempts to take a shared lock on byte 1. This is how we implement
1247     * single writer, multi reader semantics.
1248     *
1249     * The cleaner, each time it is invoked, attempts to take an exclusive lock
1250     * on byte 1. The owning process already either has an exclusive lock on
1251     * byte 0, or a shared lock on byte 1. This will necessarily conflict with
1252     * any shared locks on byte 1, even if it's in the same process and there
1253     * are no other holders of that shared lock. So if there is only one
1254     * read-only process, it will have byte 1 for shared access, and the
1255     * cleaner can not run in it because it will attempt to get an exclusive
1256     * lock on byte 1 (which is already locked for shared access by itself).
1257     * If a write process comes along and tries to run the cleaner, it will
1258     * attempt to get an exclusive lock on byte 1. If there are no other
1259     * reader processes (with shared locks on byte 1), and no other writers
1260     * (which are running cleaners on with exclusive locks on byte 1), then the
1261     * cleaner will run.
1262     */
1263    public boolean lockEnvironment(boolean readOnly, boolean exclusive)
1264        throws DatabaseException {
1265
1266        try {
1267        if (checkEnvHomePermissions(readOnly)) {
1268        return true;
1269        }
1270
1271            if (lockFile == null) {
1272                lockFile =
1273                    new RandomAccessFile (new File (dbEnvHome, LOCK_FILE),
1274                                 FileMode.READWRITE_MODE.getModeValue());
1275
1276            }
1277            channel = lockFile.getChannel();
1278
1279            boolean throwIt = false;
1280            try {
1281                if (exclusive) {
1282
1283                    /*
1284                     * To lock exclusive, must have exclusive on
1285                     * shared reader area (byte 1).
1286                     */
1287                    exclLock = channel.tryLock(1, 1, false);
1288                    if (exclLock == null) {
1289                        return false;
1290                    }
1291                    return true;
1292                } else {
1293                    if (readOnly) {
1294                        envLock = channel.tryLock(1, 1, true);
1295                    } else {
1296                        envLock = channel.tryLock(0, 1, false);
1297                    }
1298                    if (envLock == null) {
1299                        throwIt = true;
1300                    }
1301                }
1302            } catch (OverlappingFileLockException e) {
1303                throwIt = true;
1304            }
1305            if (throwIt) {
1306                throw new LogException
1307                    ("A " + LOCK_FILE + " file exists in " +
1308                     dbEnvHome.getAbsolutePath() +
1309                     " The environment can not be locked for " +
1310                     (readOnly ? "shared" : "single writer") + " access.");
1311            }
1312        } catch (IOException IOE) {
1313            throw new LogException(IOE.toString());
1314        }
1315        return true;
1316    }
1317
1318    public void releaseExclusiveLock()
1319        throws DatabaseException {
1320
1321        try {
1322            if (exclLock != null) {
1323                exclLock.release();
1324            }
1325        } catch (IOException IOE) {
1326            throw new DatabaseException(IOE);
1327        }
1328    }
1329
1330    /**
1331     * Ensure that if the environment home dir is on readonly media or in a
1332     * readonly directory that the environment has been opened for readonly
1333     * access.
1334     *
1335     * @return true if the environment home dir is readonly.
1336     */
1337    public boolean checkEnvHomePermissions(boolean readOnly)
1338    throws DatabaseException {
1339
1340    boolean envDirIsReadOnly = !dbEnvHome.canWrite();
1341    if (envDirIsReadOnly && !readOnly) {
1342
1343            /*
1344             * Use the absolute path in the exception message, to
1345             * make a mis-specified relative path problem more obvious.
1346             */
1347        throw new DatabaseException
1348        ("The Environment directory " +
1349                 dbEnvHome.getAbsolutePath() +
1350                 " is not writable, but the " +
1351         "Environment was opened for read-write access.");
1352    }
1353
1354    return envDirIsReadOnly;
1355    }
1356
1357    /**
1358     * Truncate a log at this position. Used by recovery to a timestamp
1359     * utilities and by recovery to set the end-of-log position.
1360     *
1361     * <p>This method forces a new log file to be written next, if the last
1362     * file (the file truncated to) has an old version in its header. This
1363     * ensures that when the log is opened by an old version of JE, a version
1364     * incompatibility will be detected. [#11243]</p>
1365     */
1366    public void truncateLog(long fileNum, long offset)
1367        throws IOException , DatabaseException {
1368
1369        FileHandle handle = makeFileHandle(fileNum, FileMode.READWRITE_MODE);
1370        RandomAccessFile file = handle.getFile();
1371
1372        try {
1373            file.getChannel().truncate(offset);
1374        } finally {
1375            file.close();
1376        }
1377
1378        if (handle.isOldHeaderVersion()) {
1379            forceNewFile = true;
1380        }
1381    }
1382
1383    /**
1384     * Set the flag that causes a new file to be written before the next write.
1385     */
1386    void forceNewLogFile() {
1387    forceNewFile = true;
1388    }
1389
1390    /**
1391     * Return the offset of the first log entry after the file header.
1392     */
1393
1394    /**
1395     * @return the size in bytes of the file header log entry.
1396     */
1397    public static int firstLogEntryOffset() {
1398        return FileHeader.entrySize() + LogManager.HEADER_BYTES;
1399    }
1400
1401    /**
1402     * Return the next available LSN in the log. Note that this is
1403     * unsynchronized, so is only valid as an approximation of log size.
1404     */
1405    public long getNextLsn() {
1406        return nextAvailableLsn;
1407    }
1408
1409    /**
1410     * Return the last allocated LSN in the log. Note that this is
1411     * unsynchronized, so if it is called outside the log write latch it is
1412     * only valid as an approximation of log size.
1413     */
1414    public long getLastUsedLsn() {
1415        return lastUsedLsn;
1416    }
1417
1418    /*
1419     * fsync stats.
1420     */
1421    public long getNFSyncs() {
1422        return syncManager.getNFSyncs();
1423    }
1424
1425    public long getNFSyncRequests() {
1426        return syncManager.getNFSyncRequests();
1427    }
1428
1429    public long getNFSyncTimeouts() {
1430        return syncManager.getNTimeouts();
1431    }
1432
1433    void loadStats(StatsConfig config, EnvironmentStats stats)
1434        throws DatabaseException {
1435
1436        syncManager.loadStats(config, stats);
1437    }
1438
1439    /*
1440     * Unit test support
1441     */
1442
1443    /*
1444     * @return ids of files in cache
1445     */
1446    Set getCacheKeys() {
1447        return fileCache.getCacheKeys();
1448    }
1449
1450    /**
1451     * Clear a file out of the file cache regardless of mode type.
1452     */
1453    private void clearFileCache(long fileNum)
1454        throws IOException , DatabaseException {
1455
1456        fileCacheLatch.acquire();
1457        try {
1458            fileCache.remove(fileNum);
1459        } finally {
1460            fileCacheLatch.release();
1461        }
1462    }
1463
1464    /*
1465     * The file cache keeps N RandomAccessFile objects cached for file
1466     * access. The cache consists of two parts: a Hashtable that doesn't
1467     * require extra synchronization, for the most common access, and a linked
1468     * list of files to support cache administration. Looking up a file from
1469     * the hash table doesn't require extra latching, but adding or deleting a
1470     * file does.
1471     */
1472    private static class FileCache {
1473        private Map fileMap; // Long->file
1474 private LinkedList fileList; // list of file numbers
1475 private int fileCacheSize;
1476
1477        FileCache(DbConfigManager configManager)
1478            throws DatabaseException {
1479
1480            /*
1481             * A fileMap maps the file number to FileHandles (RandomAccessFile,
1482             * latch). The fileList is a list of Longs to determine which files
1483             * to eject out of the file cache if it's too small.
1484             */
1485            fileMap = new Hashtable ();
1486            fileList = new LinkedList ();
1487            fileCacheSize =
1488        configManager.getInt(EnvironmentParams.LOG_FILE_CACHE_SIZE);
1489        }
1490
1491        private FileHandle get(Long fileId) {
1492            return (FileHandle) fileMap.get(fileId);
1493        }
1494
1495        private void add(Long fileId, FileHandle fileHandle)
1496            throws DatabaseException {
1497
1498            /*
1499             * Does the cache have any room or do we have to evict? Hunt down
1500             * the file list for an unused file. Note that the file cache might
1501             * actually grow past the prescribed size if there is nothing
1502             * evictable. Should we try to shrink the file cache? Presently if
1503             * it grows, it doesn't shrink.
1504             */
1505            if (fileList.size() >= fileCacheSize) {
1506                Iterator iter = fileList.iterator();
1507                while (iter.hasNext()) {
1508                    Long evictId = (Long ) iter.next();
1509                    FileHandle evictTarget = (FileHandle) fileMap.get(evictId);
1510
1511                    /*
1512                     * Try to latch. If latchNoWait returns false, then another
1513                     * thread owns this latch. Note that a thread that's trying
1514                     * to get a new file handle should never already own the
1515                     * latch on another file handle, because these latches are
1516                     * meant to be short lived and only held over the i/o out
1517                     * of the file.
1518                     */
1519                    if (evictTarget.latchNoWait()) {
1520                        try {
1521                            fileMap.remove(evictId);
1522                            iter.remove();
1523                            evictTarget.close();
1524                        } catch (IOException e) {
1525                            throw new DatabaseException (e);
1526                        } finally {
1527                            evictTarget.release();
1528                        }
1529                        break;
1530                    }
1531                }
1532            }
1533
1534            /*
1535             * We've done our best to evict. Add the file the the cache now
1536             * whether or not we did evict.
1537             */
1538            fileList.add(fileId);
1539            fileMap.put(fileId, fileHandle);
1540        }
1541
1542        /**
1543         * Take any file handles corresponding to this file name out of the
1544         * cache. A file handle could be there twice, in rd only and in r/w
1545         * mode.
1546         */
1547        private void remove(long fileNum)
1548            throws IOException , DatabaseException {
1549
1550            Iterator iter = fileList.iterator();
1551            while (iter.hasNext()) {
1552                Long evictId = (Long ) iter.next();
1553                if (evictId.longValue() == fileNum) {
1554                    FileHandle evictTarget = (FileHandle) fileMap.get(evictId);
1555                    try {
1556                        evictTarget.latch();
1557                        fileMap.remove(evictId);
1558                        iter.remove();
1559                        evictTarget.close();
1560                    } finally {
1561                        evictTarget.release();
1562                    }
1563                }
1564            }
1565        }
1566
1567        private void clear()
1568            throws IOException , DatabaseException {
1569
1570            Iterator iter = fileMap.values().iterator();
1571            while (iter.hasNext()) {
1572                FileHandle fileHandle = (FileHandle) iter.next();
1573                try {
1574                    fileHandle.latch();
1575                    fileHandle.close();
1576                    iter.remove();
1577                } finally {
1578                    fileHandle.release();
1579                }
1580            }
1581            fileMap.clear();
1582            fileList.clear();
1583        }
1584
1585        private Set getCacheKeys() {
1586            return fileMap.keySet();
1587        }
1588    }
1589
1590    /**
1591     * The LogEndFileDescriptor is used to write and fsync the end of the log.
1592     * Because the JE log is append only, there is only one logical R/W file
1593     * descriptor for the whole environment. This class actually implements two
1594     * RandomAccessFile instances, one for writing and one for fsyncing, so the
1595     * two types of operations don't block each other.
1596     *
1597     * The write file descriptor is considered the master. Manipulation of
1598     * this class is done under the log write latch. Here's an explanation of
1599     * why the log write latch is sufficient to safeguard all operations.
1600     *
1601     * There are two types of callers who may use this file descriptor: the
1602     * thread that is currently writing to the end of the log and any threads
1603     * that are fsyncing on behalf of the FSyncManager.
1604     *
1605     * The writing thread appends data to the file and fsyncs the file when we
1606     * flip over to a new log file. The file is only instantiated at the point
1607     * that it must do so -- which is either when the first fsync is required
1608     * by JE or when the log file is full and we flip files. Therefore, the
1609     * writing thread has two actions that change this descriptor -- we
1610     * initialize the file descriptor for the given log file at the first write
1611     * to the file, and we close the file descriptor when the log file is full.
1612     * Therefore is a period when there is no log descriptor -- when we have
1613     * not yet written a log buffer into a given log file.
1614     *
1615     * The fsyncing threads ask for the log end file descriptor asynchronously,
1616     * but will never modify it. These threads may arrive at the point when
1617     * the file descriptor is null, and therefore skip their fysnc, but that is
1618     * fine because it means a writing thread already flipped that target file
1619     * and has moved on to the next file.
1620     *
1621     * Time Activity
1622     * 10 thread 1 writes log entry A into file 0x0, issues fsync
1623     * outside of log write latch, yields the processor
1624     * 20 thread 2 writes log entry B, piggybacks off thread 1
1625     * 30 thread 3 writes log entry C, but no room left in that file,
1626     * so it flips the log, and fsyncs file 0x0, all under the log
1627     * write latch. It nulls out endOfLogRWFile, moves onto file
1628     * 0x1, but doesn't create the file yet.
1629     * 40 thread 1 finally comes along, but endOfLogRWFile is null--
1630     * no need to fsync in that case, 0x0 got fsynced.
1631     */
1632    class LogEndFileDescriptor {
1633        private RandomAccessFile endOfLogRWFile = null;
1634        private RandomAccessFile endOfLogSyncFile = null;
1635    private Object fsyncFileSynchronizer = new Object ();
1636
1637        /**
1638         * getWritableFile must be called under the log write latch.
1639         */
1640        RandomAccessFile getWritableFile(long fileNumber)
1641            throws RunRecoveryException {
1642
1643            try {
1644
1645                if (endOfLogRWFile == null) {
1646
1647                    /*
1648                     * We need to make a file descriptor for the end of the
1649                     * log. This is guaranteed to be called under the log
1650                     * write latch.
1651                     */
1652                    endOfLogRWFile =
1653                        makeFileHandle(fileNumber,
1654                                       FileMode.READWRITE_MODE).getFile();
1655            synchronized (fsyncFileSynchronizer) {
1656            endOfLogSyncFile =
1657                makeFileHandle(fileNumber,
1658                       FileMode.READWRITE_MODE).getFile();
1659            }
1660                }
1661            
1662                return endOfLogRWFile;
1663            } catch (Exception e) {
1664
1665                /*
1666                 * If we can't get a write channel, we need to go into
1667                 * RunRecovery state.
1668                 */
1669                throw new RunRecoveryException(envImpl, e);
1670            }
1671        }
1672
1673        /**
1674         * FSync the log file that makes up the end of the log.
1675         */
1676        void force()
1677            throws DatabaseException, IOException {
1678
1679            /*
1680             * Get a local copy of the end of the log file descriptor, it could
1681             * change. No need to latch, no harm done if we get an old file
1682             * descriptor, because we forcibly fsync under the log write latch
1683             * when we switch files.
1684             *
1685             * If there is no current end file descriptor, we know that the log
1686             * file has flipped to a new file since the fsync was issued.
1687             */
1688        synchronized (fsyncFileSynchronizer) {
1689        RandomAccessFile file = endOfLogSyncFile;
1690        if (file != null) {
1691            
1692            FileChannel channel = file.getChannel();
1693            try {
1694            channel.force(false);
1695            } catch (ClosedChannelException e) {
1696
1697            /*
1698             * The channel should never be closed. It may be closed
1699             * because of an interrupt received by another
1700             * thread. See SR [#10463]
1701             */
1702            throw new RunRecoveryException
1703                (envImpl,
1704                 "Channel closed, may be due to thread interrupt",
1705                 e);
1706            }
1707            
1708            assert EnvironmentImpl.maybeForceYield();
1709        }
1710        }
1711        }
1712
1713        /**
1714         * Close the end of the log file descriptor. Use atomic assignment to
1715         * ensure that we won't force and close on the same descriptor.
1716         */
1717        void close()
1718            throws IOException {
1719
1720            IOException firstException = null;
1721            if (endOfLogRWFile != null) {
1722                RandomAccessFile file = endOfLogRWFile;
1723
1724                /*
1725                 * Null out so that other threads know endOfLogRWFile is no
1726                 * longer available.
1727                 */
1728                endOfLogRWFile = null;
1729                try {
1730                    file.close();
1731                } catch (IOException e) {
1732                    /* Save this exception, so we can try the second close. */
1733                    firstException = e;
1734                }
1735            }
1736        synchronized (fsyncFileSynchronizer) {
1737        if (endOfLogSyncFile != null) {
1738            RandomAccessFile file = endOfLogSyncFile;
1739
1740            /*
1741             * Null out so that other threads know endOfLogSyncFile is
1742             * no longer available.
1743             */
1744            endOfLogSyncFile = null;
1745            file.close();
1746        }
1747
1748        if (firstException != null) {
1749            throw firstException;
1750        }
1751        }
1752        }
1753    }
1754
1755    /*
1756     * Generate IOExceptions for testing.
1757     */
1758
1759    /* Testing switch. */
1760    static boolean RUNRECOVERY_EXCEPTION_TESTING = false;
1761    /* Max write counter value. */
1762    private static final int RUNRECOVERY_EXCEPTION_MAX = 100;
1763    /* Current write counter value. */
1764    private int runRecoveryExceptionCounter = 0;
1765    /* Whether an exception has been thrown. */
1766    private boolean runRecoveryExceptionThrown = false;
1767    /* Random number generator. */
1768    private Random runRecoveryExceptionRandom = null;
1769
1770    private void generateRunRecoveryException(RandomAccessFile file,
1771                                              ByteBuffer data,
1772                                              long destOffset)
1773        throws DatabaseException, IOException {
1774
1775        if (runRecoveryExceptionThrown) {
1776            try {
1777                throw new Exception ("Write after RunRecoveryException");
1778            } catch (Exception e) {
1779                e.printStackTrace();
1780            }
1781        }
1782        runRecoveryExceptionCounter += 1;
1783        if (runRecoveryExceptionCounter >= RUNRECOVERY_EXCEPTION_MAX) {
1784            runRecoveryExceptionCounter = 0;
1785        }
1786        if (runRecoveryExceptionRandom == null) {
1787            runRecoveryExceptionRandom = new Random (System.currentTimeMillis());
1788        }
1789        if (runRecoveryExceptionCounter ==
1790            runRecoveryExceptionRandom.nextInt(RUNRECOVERY_EXCEPTION_MAX)) {
1791            int len = runRecoveryExceptionRandom.nextInt(data.remaining());
1792            if (len > 0) {
1793                byte[] a = new byte[len];
1794                data.get(a, 0, len);
1795                ByteBuffer buf = ByteBuffer.wrap(a);
1796                writeToFile(file, buf, destOffset);
1797            }
1798            runRecoveryExceptionThrown = true;
1799        throw new RunRecoveryException
1800            (envImpl, "Randomly generated for testing");
1801        }
1802    }
1803}
1804

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags