DataFileCache


1   /* Copyright (c) 2001-2005, The HSQL Development Group
2    * All rights reserved.
3    *
4    * Redistribution and use in source and binary forms, with or without
5    * modification, are permitted provided that the following conditions are met:
6    *
7    * Redistributions of source code must retain the above copyright notice, this
8    * list of conditions and the following disclaimer.
9    *
10   * Redistributions in binary form must reproduce the above copyright notice,
11   * this list of conditions and the following disclaimer in the documentation
12   * and/or other materials provided with the distribution.
13   *
14   * Neither the name of the HSQL Development Group nor the names of its
15   * contributors may be used to endorse or promote products derived from this
16   * software without specific prior written permission.
17   *
18   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19   * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21   * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
22   * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23   * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24   * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26   * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28   * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29   */
30  
31  
32  package org.hsqldb.persist;
33  
34  import java.io.File  ;
35  import java.io.IOException  ;
36  
37  import org.hsqldb.Database;
38  import org.hsqldb.HsqlException;
39  import org.hsqldb.Trace;
40  import org.hsqldb.lib.FileAccess;
41  import org.hsqldb.lib.FileUtil;
42  import org.hsqldb.lib.SimpleLog;
43  import org.hsqldb.lib.StopWatch;
44  import org.hsqldb.lib.Storage;
45  import org.hsqldb.lib.ZipUnzipFile;
46  import org.hsqldb.rowio.RowInputBinary;
47  import org.hsqldb.rowio.RowInputInterface;
48  import org.hsqldb.rowio.RowOutputBinary;
49  import org.hsqldb.rowio.RowOutputInterface;
50  import org.hsqldb.store.BitMap;
51  
52  /**
53   * Acts as a manager for CACHED table persistence.<p>
54   *
55   * This contains the top level functionality. Provides file management services
56   * and access.<p>
57   *
58   * Rewritten for 1.8.0 together with Cache.
59   *
60   * @author fredt@users
61   * @version 1.8.0
62   * @since 1.7.2
63   */
64  public class DataFileCache {
65  
66      protected FileAccess fa;
67  
68      // flags
69      public static final int FLAG_ISSAVED = 2;
70      public static final int FLAG_ROWINFO = 3;
71  
72      // file format fields
73      static final int LONG_EMPTY_SIZE      = 4;     // empty space size
74      static final int LONG_FREE_POS_POS    = 12;    // where iFreePos is saved
75      static final int LONG_EMPTY_INDEX_POS = 20;    // empty space index
76      static final int FLAGS_POS            = 28;
77      static final int INITIAL_FREE_POS     = 32;
78  
79      //
80      DataFileBlockManager     freeBlocks;
81      private static final int initIOBufferSize = 256;
82  
83      //
84      protected String     fileName;
85      protected String     backupFileName;
86      protected Database database;
87  
88      // this flag is used externally to determine if a backup is required
89      protected boolean fileModified;
90      protected int     cacheFileScale;
91  
92      // post openning constant fields
93      protected boolean cacheReadonly;
94  
95      // cache operation mode
96      protected boolean storeOnInsert;
97  
98      //
99      protected int     cachedRowPadding = 8;
100     protected boolean hasRowInfo       = false;
101 
102     // reusable input / output streams
103     protected RowInputInterface  rowIn;
104     protected RowOutputInterface rowOut;
105 
106     //
107     public long maxDataFileSize;
108 
109     //
110     protected Storage dataFile;
111     protected long    fileFreePosition;
112     protected int     maxCacheSize;                // number of Rows
113     protected long    maxCacheBytes;               // number of bytes
114     protected int     maxFreeBlocks;
115     protected Cache   cache;
116 
117     public DataFileCache(Database db,
118                          String   baseFileName) throws HsqlException {
119 
120         initParams(db, baseFileName);
121 
122         cache = new Cache(this);
123     }
124 
125     /**
126      * initial external parameters are set here.
127      */
128     protected void initParams(Database database,
129                               String   baseFileName) throws HsqlException {
130 
131         HsqlDatabaseProperties props = database.getProperties();
132 
133         fileName       = baseFileName + ".data";
134         backupFileName = baseFileName + ".backup";
135         this.database  = database;
136         fa             = database.getFileAccess();
137 
138         int cacheScale = props.getIntegerProperty(
139             HsqlDatabaseProperties.hsqldb_cache_scale, 14, 8, 18);
140         int cacheSizeScale = props.getIntegerProperty(
141             HsqlDatabaseProperties.hsqldb_cache_size_scale, 10, 6, 20);
142         int cacheFreeCountScale = props.getIntegerProperty(
143             HsqlDatabaseProperties.hsqldb_cache_free_count_scale, 9, 6, 12);
144 
145         cacheFileScale = database.getProperties().getIntegerProperty(
146             HsqlDatabaseProperties.hsqldb_cache_file_scale, 1);
147 
148         if (cacheFileScale != 1) {
149             cacheFileScale = 8;
150         }
151 
152         cacheReadonly = database.isFilesReadOnly();
153 
154         int lookupTableLength = 1 << cacheScale;
155         int avgRowBytes       = 1 << cacheSizeScale;
156 
157         maxCacheSize    = lookupTableLength * 3;
158         maxCacheBytes   = maxCacheSize * avgRowBytes;
159         maxDataFileSize = cacheFileScale == 1 ? Integer.MAX_VALUE
160                                               : (long) Integer.MAX_VALUE * 4;
161         maxFreeBlocks   = 1 << cacheFreeCountScale;
162         dataFile        = null;
163     }
164 
165     /**
166      * Opens the *.data file for this cache, setting the variables that
167      * allow access to the particular database version of the *.data file.
168      */
169     public void open(boolean readonly) throws HsqlException {
170 
171         fileFreePosition = 0;
172 
173         database.logger.appLog.logContext(SimpleLog.LOG_NORMAL, "start");
174 
175         try {
176             boolean preexists = database.isFilesInJar();
177             long    freesize  = 0;
178 
179             if (!preexists && fa.isStreamElement(fileName)) {
180                 if (database.isStoredFileAccess()) {
181                     preexists = true;
182                 } else {
183 
184                     // discard "empty" databases
185                     File   f = new File  (fileName);
186 
187                     preexists = f.length() > INITIAL_FREE_POS;
188                 }
189             }
190 
191             if (preexists) {
192                 String   version = database.getProperties().getProperty(
193                     HsqlDatabaseProperties.hsqldb_cache_version);
194                 boolean v17 =
195                     HsqlDatabaseProperties.VERSION_STRING_1_7_0.equals(
196                         version);
197 
198                 // for later versions
199                 boolean v18 =
200                     HsqlDatabaseProperties.VERSION_STRING_1_8_0.equals(
201                         version);
202 
203                 if (!v17) {
204                     throw Trace.error(Trace.WRONG_DATABASE_FILE_VERSION);
205                 }
206             }
207 
208             boolean isNio = database.getProperties().isPropertyTrue(
209                 HsqlDatabaseProperties.hsqldb_nio_data_file);
210             int fileType = isNio ? ScaledRAFile.DATA_FILE_NIO
211                                  : ScaledRAFile.DATA_FILE_RAF;
212 
213             if (database.isFilesInJar()) {
214                 fileType = ScaledRAFile.DATA_FILE_JAR;
215             }
216 
217             // oj@openofice.org - change to file access api
218             String   cname =
219                 database.getURLProperties().getProperty("storage_class_name");
220             String   skey =
221                 database.getURLProperties().getProperty("storage_key");
222 
223             dataFile = ScaledRAFile.newScaledRAFile(database, fileName,
224                     readonly, fileType, cname, skey);
225 
226             if (preexists) {
227                 dataFile.seek(FLAGS_POS);
228 
229                 int flags = dataFile.readInt();
230 
231                 hasRowInfo = BitMap.isSet(flags, FLAG_ROWINFO);
232 
233                 dataFile.seek(LONG_EMPTY_SIZE);
234 
235                 freesize = dataFile.readLong();
236 
237                 dataFile.seek(LONG_FREE_POS_POS);
238 
239                 fileFreePosition = dataFile.readLong();
240 
241                 if (fileFreePosition < INITIAL_FREE_POS) {
242                     fileFreePosition = INITIAL_FREE_POS;
243                 }
244             } else {
245                 fileFreePosition = INITIAL_FREE_POS;
246 
247                 dataFile.seek(LONG_FREE_POS_POS);
248                 dataFile.writeLong(INITIAL_FREE_POS);
249 
250                 // set unsaved flag;
251                 dataFile.seek(FLAGS_POS);
252                 dataFile.writeInt(0);
253             }
254 
255             initBuffers();
256 
257             fileModified = false;
258             freeBlocks = new DataFileBlockManager(maxFreeBlocks,
259                                                   cacheFileScale, freesize);
260 
261             database.logger.appLog.logContext(SimpleLog.LOG_NORMAL, "end");
262         } catch (Throwable   e) {
263             database.logger.appLog.logContext(e, "failed");
264             close(false);
265 
266             throw Trace.error(Trace.FILE_IO_ERROR, Trace.DataFileCache_open,
267                               new Object  [] {
268                 e, fileName
269             });
270         }
271     }
272 
273     /**
274      *  Parameter write indicates either an orderly close, or a fast close
275      *  without backup.
276      *
277      *  When false, just closes the file.
278      *
279      *  When true, writes out all cached rows that have been modified and the
280      *  free position pointer for the *.data file and then closes the file.
281      */
282     public void close(boolean write) throws HsqlException {
283 
284         SimpleLog appLog = database.logger.appLog;
285 
286         try {
287             if (cacheReadonly) {
288                 if (dataFile != null) {
289                     dataFile.close();
290                 }
291 
292                 return;
293             }
294 
295             StopWatch sw = new StopWatch();
296 
297             appLog.sendLine(SimpleLog.LOG_NORMAL,
298                             "DataFileCache.close(" + write + ") : start");
299 
300             if (write) {
301                 cache.saveAll();
302                 Trace.printSystemOut("saveAll: " + sw.elapsedTime());
303                 appLog.sendLine(SimpleLog.LOG_NORMAL,
304                                 "DataFileCache.close() : save data");
305 
306                 if (fileModified || freeBlocks.isModified()) {
307 
308                     // set empty
309                     dataFile.seek(LONG_EMPTY_SIZE);
310                     dataFile.writeLong(freeBlocks.getLostBlocksSize());
311 
312                     // set end
313                     dataFile.seek(LONG_FREE_POS_POS);
314                     dataFile.writeLong(fileFreePosition);
315 
316                     // set saved flag;
317                     dataFile.seek(FLAGS_POS);
318 
319                     int flag = BitMap.set(0, FLAG_ISSAVED);
320 
321                     if (hasRowInfo) {
322                         flag = BitMap.set(flag, FLAG_ROWINFO);
323                     }
324 
325                     dataFile.writeInt(flag);
326                     appLog.sendLine(SimpleLog.LOG_NORMAL,
327                                     "DataFileCache.close() : flags");
328 
329                     //
330                     if (dataFile.length() != fileFreePosition) {
331                         dataFile.seek(fileFreePosition);
332                     }
333 
334                     appLog.sendLine(SimpleLog.LOG_NORMAL,
335                                     "DataFileCache.close() : seek end");
336                     Trace.printSystemOut("pos and flags: "
337                                          + sw.elapsedTime());
338                 }
339             }
340 
341             if (dataFile != null) {
342                 dataFile.close();
343                 appLog.sendLine(SimpleLog.LOG_NORMAL,
344                                 "DataFileCache.close() : close");
345 
346                 dataFile = null;
347 
348                 Trace.printSystemOut("close: " + sw.elapsedTime());
349             }
350 
351             boolean empty = fileFreePosition == INITIAL_FREE_POS;
352 
353             if (empty) {
354                 fa.removeElement(fileName);
355                 fa.removeElement(backupFileName);
356             }
357         } catch (Throwable   e) {
358             appLog.logContext(e, null);
359 
360             throw Trace.error(Trace.FILE_IO_ERROR, Trace.DataFileCache_close,
361                               new Object  [] {
362                 e, fileName
363             });
364         }
365     }
366 
367     protected void initBuffers() {
368 
369         if (rowOut == null
370                 || ((RowOutputBinary) rowOut).getBuffer().length
371                    > initIOBufferSize) {
372             rowOut = new RowOutputBinary(256);
373         }
374 
375         if (rowIn == null
376                 || ((RowInputBinary) rowIn).getBuffer().length
377                    > initIOBufferSize) {
378             rowIn = new RowInputBinary(new byte[256]);
379         }
380     }
381 
382     /**
383      *  Writes out all the rows to a new file without fragmentation.
384      */
385     public void defrag() throws HsqlException {
386 
387         if (cacheReadonly) {
388             return;
389         }
390 
391         if (fileFreePosition == INITIAL_FREE_POS) {
392             return;
393         }
394 
395         database.logger.appLog.logContext(SimpleLog.LOG_NORMAL, "start");
396 
397         try {
398             boolean wasNio = dataFile.wasNio();
399 
400             cache.saveAll();
401 
402             DataFileDefrag dfd = new DataFileDefrag(database, this, fileName);
403 
404             dfd.process();
405             close(false);
406             deleteFile(wasNio);
407             renameDataFile();
408             backupFile();
409             database.getProperties().setProperty(
410                 HsqlDatabaseProperties.hsqldb_cache_version,
411                 HsqlDatabaseProperties.VERSION_STRING_1_7_0);
412             database.getProperties().save();
413             cache.clear();
414 
415             cache = new Cache(this);
416 
417             open(cacheReadonly);
418             dfd.updateTableIndexRoots();
419             dfd.updateTransactionRowIDs();
420         } catch (Throwable   e) {
421             database.logger.appLog.logContext(e, null);
422 
423             throw new HsqlException(
424                 e, Trace.getMessage(Trace.GENERAL_IO_ERROR),
425                 Trace.GENERAL_IO_ERROR);
426         }
427 
428         database.logger.appLog.logContext(SimpleLog.LOG_NORMAL, "end");
429     }
430 
431     /**
432      * Used when a row is deleted as a result of some DML or DDL command.
433      * Removes the row from the cache data structures.
434      * Adds the file space for the row to the list of free positions.
435      */
436     public synchronized void remove(int i,
437                                     PersistentStore store)
438                                     throws IOException   {
439 
440         CachedObject r    = release(i);
441         int          size = r == null ? getStorageSize(i)
442                                       : r.getStorageSize();
443 
444         freeBlocks.add(i, size);
445     }
446 
447     public synchronized void removePersistence(int i,
448             PersistentStore store) throws IOException   {}
449 
450     /**
451      * Allocates file space for the row. <p>
452      *
453      * Free space is requested from the block manager if it exists.
454      * Otherwise the file is grown to accommodate it.
455      */
456     private int setFilePos(CachedObject r) throws IOException   {
457 
458         int rowSize = r.getStorageSize();
459         int i       = freeBlocks == null ? -1
460                                          : freeBlocks.get(rowSize);
461 
462         if (i == -1) {
463             i = (int) (fileFreePosition / cacheFileScale);
464 
465             long newFreePosition = fileFreePosition + rowSize;
466 
467             if (newFreePosition > maxDataFileSize) {
468                 throw new IOException  (
469                     Trace.getMessage(Trace.DATA_FILE_IS_FULL));
470             }
471 
472             fileFreePosition = newFreePosition;
473         }
474 
475         r.setPos(i);
476 
477         return i;
478     }
479 
480     public synchronized void add(CachedObject object) throws IOException   {
481 
482         int size = object.getRealSize(rowOut);
483 
484         size = ((size + cachedRowPadding - 1) / cachedRowPadding)
485                * cachedRowPadding;
486 
487         object.setStorageSize(size);
488 
489         int i = setFilePos(object);
490 
491         cache.put(i, object);
492 
493         // was previously used for text tables
494         if (storeOnInsert) {
495             saveRow(object);
496         }
497     }
498 
499     /**
500      * For a CacheObject that had been previously released from the cache.
501      * A new version is introduced, using the preallocated space for the object.
502      */
503     public synchronized void restore(CachedObject object) throws IOException   {
504 
505         int i = object.getPos();
506 
507         cache.put(i, object);
508 
509         // was previously used for text tables
510         if (storeOnInsert) {
511             saveRow(object);
512         }
513     }
514 
515     public synchronized int getStorageSize(int i) throws IOException   {
516 
517         CachedObject value = cache.get(i);
518 
519         if (value != null) {
520             return value.getStorageSize();
521         }
522 
523         return readSize(i);
524     }
525 
526     public synchronized CachedObject get(int i, PersistentStore store,
527                                          boolean keep) throws HsqlException {
528 
529         if (i < 0) {
530             return null;
531         }
532 
533         try {
534             CachedObject object = cache.get(i);
535 
536             if (object == null) {
537                 RowInputInterface rowInput = readObject(i);
538 
539                 if (rowInput == null) {
540                     return null;
541                 }
542 
543                 object = store.get(rowInput);
544 
545                 // for text tables with empty rows at the beginning,
546                 // pos may move forward in readObject
547                 i = object.getPos();
548 
549                 cache.put(i, object);
550             }
551 
552             if (keep) {
553                 object.keepInMemory(true);
554             }
555 
556             return object;
557         } catch (IOException   e) {
558             database.logger.appLog.logContext(e, fileName + " get pos: " + i);
559 
560             throw Trace.error(Trace.DATA_FILE_ERROR,
561                               Trace.DataFileCache_makeRow, new Object  [] {
562                 e, fileName
563             });
564         }
565     }
566 
567     synchronized RowInputInterface getRaw(int i) throws IOException   {
568         return readObject(i);
569     }
570 
571     protected synchronized int readSize(int pos) throws IOException   {
572 
573         dataFile.seek((long) pos * cacheFileScale);
574 
575         return dataFile.readInt();
576     }
577 
578     protected synchronized RowInputInterface readObject(int pos)
579     throws IOException   {
580 
581         dataFile.seek((long) pos * cacheFileScale);
582 
583         int size = dataFile.readInt();
584 
585         rowIn.resetRow(pos, size);
586         dataFile.read(rowIn.getBuffer(), 4, size - 4);
587 
588         return rowIn;
589     }
590 
591     public synchronized CachedObject release(int i) {
592         return cache.release(i);
593     }
594 
595     protected synchronized void saveRows(CachedObject[] rows, int offset,
596                                          int count) throws IOException   {
597 
598         try {
599             for (int i = offset; i < offset + count; i++) {
600                 CachedObject r = rows[i];
601 
602                 saveRow(r);
603 
604                 rows[i] = null;
605             }
606         } catch (IOException   e) {
607             database.logger.appLog.logContext(e, null);
608 
609             throw e;
610         } catch (Throwable   e) {
611             database.logger.appLog.logContext(e, null);
612 
613             throw new IOException  (e.toString());
614         } finally {
615             initBuffers();
616         }
617     }
618 
619     /**
620      * Writes out the specified Row. Will write only the Nodes or both Nodes
621      * and table row data depending on what is not already persisted to disk.
622      */
623     public synchronized void saveRow(CachedObject row) throws IOException   {
624 
625         setFileModified();
626         rowOut.reset();
627         row.write(rowOut);
628         dataFile.seek((long) row.getPos() * cacheFileScale);
629         dataFile.write(rowOut.getOutputStream().getBuffer(), 0,
630                        rowOut.getOutputStream().size());
631     }
632 
633     /**
634      *  Saves the *.data file as compressed *.backup.
635      *
636      * @throws  HsqlException
637      */
638     void backupFile() throws IOException   {
639 
640         try {
641             if (fa.isStreamElement(fileName)) {
642                 ZipUnzipFile.compressFile(fileName, backupFileName + ".new",
643                                           database.getFileAccess());
644             }
645         } catch (IOException   e) {
646             database.logger.appLog.logContext(e, null);
647 
648             throw e;
649         }
650     }
651 
652     void renameBackupFile() {
653 
654         if (fa.isStreamElement(backupFileName + ".new")) {
655             fa.removeElement(backupFileName);
656             fa.renameElement(backupFileName + ".new", backupFileName);
657         }
658     }
659 
660     /**
661      *  Renames the *.data.new file.
662      *
663      * @throws  HsqlException
664      */
665     void renameDataFile() {
666 
667         if (fa.isStreamElement(fileName + ".new")) {
668             fa.removeElement(fileName);
669             fa.renameElement(fileName + ".new", fileName);
670         }
671     }
672 
673     void deleteFile(boolean wasNio) {
674 
675         // first attemp to delete
676         fa.removeElement(fileName);
677 
678         if (fa.isStreamElement(fileName)) {
679             if (wasNio) {
680                 System.gc();
681                 fa.removeElement(fileName);
682             }
683 
684             if (fa.isStreamElement(fileName)) {
685                 fa.renameElement(fileName, fileName + ".old");
686 
687                 File   oldfile = new File  (fileName + ".old");
688 
689                 FileUtil.deleteOnExit(oldfile);
690             }
691         }
692     }
693 
694     void deleteBackup() {
695         fa.removeElement(backupFileName);
696     }
697 
698     /**
699      * This method deletes a data file or resets its free position.
700      * this is used only for nio files - not OOo files
701      */
702     static void deleteOrResetFreePos(Database database, String   filename) {
703 
704         ScaledRAFile raFile = null;
705 
706         database.getFileAccess().removeElement(filename);
707 
708         if (database.isStoredFileAccess()) {
709             return;
710         }
711 
712         if (!database.getFileAccess().isStreamElement(filename)) {
713             return;
714         }
715 
716         try {
717             raFile = new ScaledRAFile(database, filename, false);
718 
719             raFile.seek(LONG_FREE_POS_POS);
720             raFile.writeLong(INITIAL_FREE_POS);
721         } catch (IOException   e) {
722             database.logger.appLog.logContext(e, null);
723         } finally {
724             if (raFile != null) {
725                 try {
726                     raFile.close();
727                 } catch (IOException   e) {
728                     database.logger.appLog.logContext(e, null);
729                 }
730             }
731         }
732     }
733 
734     public int capacity() {
735         return maxCacheSize;
736     }
737 
738     public long bytesCapacity() {
739         return maxCacheBytes;
740     }
741 
742     public long getTotalCachedBlockSize() {
743         return cache.getTotalCachedBlockSize();
744     }
745 
746     public int getFreeBlockCount() {
747         return freeBlocks.size();
748     }
749 
750     public int getTotalFreeBlockSize() {
751         return 0;
752     }
753 
754     public long getFileFreePos() {
755         return fileFreePosition;
756     }
757 
758     public int getCachedObjectCount() {
759         return cache.size();
760     }
761 
762     public String   getFileName() {
763         return fileName;
764     }
765 
766     public boolean hasRowInfo() {
767         return hasRowInfo;
768     }
769 
770     public boolean isFileModified() {
771         return fileModified;
772     }
773 
774     protected synchronized void setFileModified() throws IOException   {
775 
776         if (!fileModified) {
777 
778             // unset saved flag;
779             dataFile.seek(FLAGS_POS);
780 
781             int flag = BitMap.set(0, FLAG_ISSAVED);
782 
783             if (hasRowInfo) {
784                 flag = BitMap.set(flag, FLAG_ROWINFO);
785             }
786 
787             dataFile.writeInt(flag);
788 
789             fileModified = true;
790         }
791     }
792 }
793
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags