Java > Open Source Codes > org > netbeans > mdr > persistence > btreeimpl > btreestorage > BtreeDatabase


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */
19 package org.netbeans.mdr.persistence.btreeimpl.btreestorage;
20
21 import java.io.*;
22 import java.text.*;
23 import java.util.*;
24
25 import org.netbeans.mdr.persistence.btreeimpl.btreeindex.*;
26 import org.netbeans.mdr.persistence.*;
27 import org.netbeans.mdr.util.Logger;
28
29 /**
30 * This is the primary index for the btree implementation of Storage.
31 * It consists of three files:
32 * <p>
33 * The data file, with an extension of ".btd", which contains the
34 * serialized versions of the Streamable objects. This is accessed via
35 * the BtreeDatafile class.
36 * <p>
37 * The index file, with an extension of ".btx", which translates MOFIDs to the
38 * corresponding offset in the data file. This is accessed via the
39 * SinglevaluedBtree class.
40 * <p>
41 * The transactional log file has the extnsion ".btb". It exists only
42 * between the time a trasaction starts to be comitted and the time it
43 * is fully comitted or rolled back.
44 * <p>
45 * The data file contains enough information to regenerate the index file if the
46 * index file is lost or damaged. If the index file is missing when the btree
47 * database is opened, it is silently regenerated.
48 * <p>
49 * A Btree storage is transactional. This is implemented using the logging
50 * feature of the FileCache. Objects stored in a Btree storage have identity
51 * in memory. That is, fetching the same object multiple times will result in
52 * multiple references to the same object, not mutilple copies. This is
53 * implemented by MDRCache.
54 * <p>
55 * Performance of the Btree can be tuned by adjusting these parameters:
56 * <p>
57 * PAGE_SIZE is the size of the pages in the file cache.
58 * <p>
59 * FILE_CACHE_SIZE in the number of pages cached in memory. Increasing this
60 * will minimize the amount of reading done from the btree files, though at
61 * the cost of decreased memory for other purposes.
62 * <p>
63 * MDR_CACHE_SIZE is the number of currently unreferenced persistent objects
64 * kept in memory. Increasing this will reduce the number of reads done from
65 * the btree storage, though again at the cost of decreased memory for other
66 * purposes.
67 * <p>
68 * MDR_CACHE_THRESHHOLD is the number of changed objects to keep in memory
69 * before saving them to disk. Increasing this will reduce the amount of disk
70 * I/O done, though yet again at the cost of decreased memory for other purposes.
71 */
72
73 public class BtreeDatabase implements
74         SinglevaluedIndex, MDRCache.OverflowHandler {
75
76     /* tuning parameters explained above */
77     static final int PAGE_SIZE = 2048;
78     static final int FILE_CACHE_SIZE = 128;
79     static final int MDR_CACHE_SIZE = 1024;
80     static final int MDR_CACHE_THRESHHOLD = 1000;
81
82     /* number of times database has been modified */
83     private int modificationLevel;
84
85     /* Name of repository */
86     private final String repositoryName;
87
88     /* Data file */
89     private BtreeDataFile dataFile;
90
91     /* Index file */
92     private SinglevaluedIndex indexFile;
93
94     /* File cache containing these two files */
95     private FileCache fileCache;
96
97     /* Cache of repository objects */
98     private MDRCache cache;
99
100     /* index of classes */
101     private CounterIndex classIndex;
102
103     /* whether the class index is dirty */
104     private boolean classIndexChanged;
105
106     /* class index type */
107     private static String CLASS_INDEX_TYPE =
108         "org.netbeans.mdr.persistence.btreeimpl.btreestorage.CounterIndex";
109
110     /* table of class names */
111     private ArrayList classes;
112
113     /* index of indexes */
114     private MofidIndex indexIndex;
115
116     /* the storage which created us */
117     private BtreeStorage myStorage;
118
119     /* true if a save error has occured */
120     private boolean saveFailed = false;
121
122     /* stores data serialized by the last @link #writeStreamable */
123     private byte [] baoStrmToBytes;
124     
125     /* Streams used to write data */
126     private ByteArrayOutputStream baoStrm = new ByteArrayOutputStream();
127     private DataOutputStream daoStrm = new DataOutputStream(baoStrm);
128     
129     private TransactionCache transactionCache = null;
130             
131     /* Logging stream. Note -- this should be replaced by the standard
132     ** logger
133     */
134     private PrintStream loggingStream;
135
136     /** create a BtreeDatabase and create/open the files
137     * @param name name of database, which is used to name the files
138     * @param parent the Storage for which this object is the primary index
139     * @param isNew true if the database is being created
140     */
141     BtreeDatabase(String name, BtreeStorage parent, boolean isNew) throws StorageException {
142         repositoryName = name;
143         myStorage = parent;
144     classes = new ArrayList();
145     try {
146         classes.add(Class.forName(CLASS_INDEX_TYPE));
147     }
148     catch (ClassNotFoundException ex) {
149         throw new RuntimeException (ex.getMessage());
150     }
151         open(isNew);
152     }
153
154     static final int DFL = 0;
155     static final int IFL = 1;
156     static final int LFL = 2;
157     private static final String [] SUFFIXES = {".btd", ".btx", ".btb"};
158
159     /* construct names of repository files */
160     private static String [] getFileNames(String base) {
161         String names[] = new String [] {getFileName(base, 0), getFileName(base, 1), getFileName(base, 2)};
162     return names;
163     }
164     
165     static String getFileName(String base, int type) {
166         return base.concat(SUFFIXES[type]);
167     }
168
169     /** See if repository currenly exists */
170     static boolean exists(String base) {
171         String names[] = getFileNames(base);
172
173     return
174         (new File(names[DFL])).exists() ||
175         (new File(names[IFL])).exists() ||
176         (new File(names[LFL])).exists();
177     }
178
179     /** delete the btree repository.
180     * @return true if, at method end, there is no repository. false if
181     * the repository exists but could not be deleted for any reason
182     */
183     static boolean delete(String base) {
184         String names[] = getFileNames(base);
185         return
186         deleteFile(names[DFL]) &
187         deleteFile(names[IFL]) &
188         deleteFile(names[LFL]);
189     }
190     
191     /* Delete a file */
192     private static boolean deleteFile(String name) {
193         File file = new File(name);
194         if (!file.exists())
195             return true;
196         else
197             return file.delete();
198     }
199
200     /* Rename database */
201     static void rename(String from, String to) throws StorageException {
202     if (exists(to)) {
203         throw new StorageBadRequestException(
204             MessageFormat.format("Btree repository {0} already exists",
205             new Object [] {to} ));
206     }
207
208     String fromNames[] = getFileNames(from);
209     String toNames[] = getFileNames(to);
210     boolean success =
211         (new File(fromNames[DFL])).renameTo(new File(toNames[DFL]));
212     if (success) {
213         success =
214             (new File(fromNames[IFL])).renameTo(new File(toNames[IFL]));
215     }
216     if (success) {
217         File fromLog = new File(fromNames[LFL]);
218         if (fromLog.exists())
219         success = fromLog.renameTo(new File(toNames[DFL]));
220     }
221     if (!success) {
222         throw new StorageBadRequestException(
223         MessageFormat.format(
224             "Unable to rename btree repository {0} to {1}",
225             new Object [] {from, to} ) );
226     }
227     }
228     
229     private int getIntProperty(String propertyName, int defaultValue) {
230         String value = (String ) myStorage.getProperty(propertyName);
231         int result = defaultValue;
232         if (value != null) {
233             try {
234                 result = Integer.parseInt(value);
235             } catch (NumberFormatException e) {
236                 Logger.getDefault().log("Error getting value of " + propertyName + " storage property: " + e.getMessage());
237             }
238         }
239         return result;
240     }
241         
242     // open or create the database, depending on the value of isNew
243 private void open(boolean isNew) throws StorageException {
244         boolean rebuildIndex = false;
245
246         cache = new MDRCache(getIntProperty(BtreeFactory.CACHE_SIZE, MDR_CACHE_SIZE), this, getIntProperty(BtreeFactory.CACHE_THRESHHOLD, MDR_CACHE_THRESHHOLD), (Map) myStorage.getProperty(BtreeFactory.CACHE_INSTANCE));
247         transactionCache = new TransactionCache ();
248     String names[] = getFileNames(repositoryName);
249
250         String fileNames[] = {names[DFL], names[IFL]};
251         if (isNew) {
252             FileHeader hdr =
253                 FileHeader.createFiles(new String [] {names[IFL]},
254                     PAGE_SIZE, false);
255             BtreeDataFile.create(this.myStorage, names[DFL], hdr, PAGE_SIZE, false);
256         }
257     else {
258         File dfl = new File(names[DFL]);
259         File ifl = new File(names[IFL]);
260
261         if (dfl.exists() && ! ifl.exists()) {
262         /* Create empty index file */
263         FileHeader hdr = null;
264         try {
265             RandomAccessFile draf = new RandomAccessFile(dfl, "r");
266             hdr = new FileHeader(draf);
267             draf.close();
268         }
269         catch (IOException ex) {
270             throw new StorageIOException(ex);
271         }
272
273         hdr.addFiles(new String [] {names[IFL]}, PAGE_SIZE, false);
274             rebuildIndex = true;
275         }
276     }
277
278         fileCache = new FileCache(fileNames, repositoryName);
279         boolean failure = true;
280         try {
281             dataFile = new BtreeDataFile(this.myStorage, fileCache, 0);
282             myStorage.gen = dataFile;
283
284             indexFile = new PrimaryIndex(repositoryName, 1, fileCache, dataFile.initialMofIdConter(),
285                                 Storage.EntryType.MOFID, Storage.EntryType.INT);
286
287             if (rebuildIndex) {
288                 rebuildIndexFile();
289                 save (true);
290             }
291
292             fetchClassIndex();
293
294             if (isNew) {
295                 save (true);
296             }
297             failure = false;
298         } finally {
299             if (failure) fileCache.abort();
300         }
301     }
302
303     /** Make a copy of the database. Note that since the copy is done
304     * record by record rather than byte by byte, the data file will
305     * be compressed.
306     * @param target name for copied database
307     */
308     void copy(String target) throws StorageException {
309     FileCache copyCache = null;
310     BtreeDataFile copyFile;
311
312     if (cache.getModStatus() != 0) {
313         throw new StorageBadRequestException(
314             MessageFormat.format(
315     "There are changes to repository {0} that have not been committed",
316             new Object [] {target} ));
317     }
318
319     if (exists(target)) {
320         throw new StorageBadRequestException(
321             MessageFormat.format("Btree repository {0} already exists",
322             new Object [] {target} ));
323     }
324     String copyNames[] = getFileNames(target);
325         String fileNames[] = {copyNames[DFL]};
326
327     BtreeDataFile.create( this.myStorage, copyNames[DFL], new FileHeader(), PAGE_SIZE, false);
328     try {
329         copyCache = new FileCache(fileNames, target);
330         copyFile = new BtreeDataFile(this.myStorage, copyCache, 0);
331         dataFile.copy(copyFile);
332         copyCache.commit();
333         copyCache.close();
334
335         // Open btree database to create index file
336 BtreeDatabase db = new BtreeDatabase(target, myStorage, false);
337         db.close();
338     }
339     catch (StorageException ex) {
340         if (copyCache != null)
341         copyCache.close();
342         delete(target);
343         throw ex;
344     }
345     }
346
347     /** Compress repository. We do this by copying to a temporary repository,
348     * deleting the current repository, and renaming the temporary repository
349     * to our name.
350     */
351     public void compress() throws StorageException {
352         synchronized (myStorage) {
353             if (cache.getModStatus() != 0) {
354                 throw new StorageBadRequestException(
355                     MessageFormat.format(
356             "There are changes to repository {0} that have not been committed",
357                             new Object [] {repositoryName} ));
358             }
359
360             String tempName = "tmp" + (new Random().nextInt());
361             copy(tempName);
362             closeFiles();
363             delete(repositoryName);
364             rename(tempName, repositoryName);
365             open(false);
366         }
367     }
368     
369     /** Rebuild the index file. Since we iterate through the data file
370     * to find all of its records, all changes to the data file must
371     * already have been committed. The index file is assumed to be
372     * empty at this point.
373     */
374     private void rebuildIndexFile() throws StorageException {
375     Iterator iter = dataFile.iterator(
376                 BtreeDataFile.ITERATE_NORMAL_EXTENTS);
377     while (iter.hasNext()) {
378         NormalBtreeExtent ext = (NormalBtreeExtent)iter.next();
379             MOFID k = this.myStorage.readMOFIDData (new ByteArrayInputStream (ext.key));
380         indexFile.add(k, new Integer (ext.myChunkNum));
381     }
382     }
383         
384     /** returns the number of objects in the repository
385      *
386      */
387     public int size() {
388         synchronized (myStorage) {
389             return dataFile.size() + cache.numberNew() - cache.numberDeleted();
390         }
391     }
392
393     /** Returns the unique name of the index in the Storage.
394      * @return base file name for repository
395      */
396     public String getName() {
397         return repositoryName;
398     };
399
400     /** Returns the type of values indexed by this index.
401      * @return EntryType.STREAMABLE
402      */
403     public Storage.EntryType getValueType() {
404         return Storage.EntryType.STREAMABLE;
405     }
406
407     /** Returns the type of keys in index.
408      * @return EntryType.MOFID
409      */
410     public Storage.EntryType getKeyType() {
411         return Storage.EntryType.MOFID;
412     }
413
414     /** Close repository
415     */
416     void close() throws StorageException {
417         synchronized (myStorage) {
418             closeFiles();
419             cache.shutDown();
420             cache = null;
421             transactionCache = null;
422         }
423     }
424
425     /** Close all files
426     */
427     private void closeFiles() throws StorageException {
428         modificationLevel++;
429         fileCache.abort();
430         fileCache = null;
431         dataFile = null;
432         indexFile = null;
433     }
434
435     /** cache has reached threshhold
436     */
437     public void cacheThreshholdReached(MDRCache cach, int size)
438         throws StorageException {
439         saveChanges();
440     }
441
442     /** Commits changes to transaction cache,
443      * if cache treshold is reached, flushes cache to disk.
444      */
445     public void commitChanges() throws StorageException {
446         synchronized (myStorage) {
447             save(false);
448
449             if (transactionCache.tresholdReached ()) {
450                 try {
451                     //System.err.println("Threshhold reached!");
452 fileCache.commit();
453                     transactionCache.clear ();
454                 } catch (StorageException ex) {
455                     saveFailed = true;
456                     throw(ex);
457                 } // catch
458 } else {
459                 transactionCache.commit ();
460             }
461         }
462     }
463
464     /**
465      * Called on exit, commited data cached in transaction cache need to be written to disk.
466      */
467     public void shutDown() throws StorageException {
468         synchronized (myStorage) {
469             cache.shutDown();
470             try {
471                 fileCache.close();
472             } catch (StorageException ex) {
473                 saveFailed = true;
474                 throw(ex);
475             } // catch
476 }
477     }
478
479     /** save all changes to disk without comitting
480     */
481     public void saveChanges() throws StorageException {
482         synchronized (myStorage) {
483             save(false);
484         }
485     }
486     
487     public String toString() {
488         return myStorage.getStorageId();
489     }
490
491     /* save all changes to disk, optionally comitting */
492     private void save(boolean commit) throws StorageException {
493         MOFID id;
494         Object value;
495         
496         if (saveFailed) {
497         // If commit has failed previously, allowing this commit would
498 // save invalid data
499 throw new StorageBadRequestException(
500             "A save of this repository has failed previously. Allowing this commit to proceed would potentially corrupt persistent data.");
501     }
502
503     try {
504         modificationLevel++;
505         StorageException writeError;
506         classIndexChanged = false;
507
508             int modStatus = cache.getModStatus();
509             while (modStatus != 0) {
510                 // First, persistently delete everything that needs it
511 if ((modStatus & MDRCache.M_DELETED) != 0) {
512                     Iterator delIter = cache.getDeleted().iterator();
513                     while (delIter.hasNext()) {
514                         id = (MOFID)delIter.next();
515                         removeRecord(id);
516                         transactionCache.addDeleted(id);
517                     }
518                 }
519
520                 // Next, modify what needs modification
521 if ((modStatus & MDRCache.M_DIRTY) != 0) {
522                     Iterator dirtyIter = cache.getDirty().iterator();
523                     while (dirtyIter.hasNext()) {
524                         Map.Entry entry = (Map.Entry) dirtyIter.next();
525                         id = (MOFID) entry.getKey();
526                         value = entry.getValue();
527                         writeError = replaceRecord(id, value);
528                         if (writeError == null) {
529                             transactionCache.addReplaced(id, baoStrmToBytes);
530                         } else /*if (commit)*/ {
531                             throw writeError;
532                         }
533                     }
534                 }
535             
536                 // Last, add what needs adding
537 if ((modStatus & MDRCache.M_NEW) != 0) {
538                     Iterator newIter = cache.getNew().iterator();
539                     while (newIter.hasNext()) {
540                         Map.Entry entry = (Map.Entry) newIter.next();
541                         id = (MOFID) entry.getKey();
542                         value = entry.getValue();
543                         writeError = addRecord(id, value);
544                         if (writeError == null) {
545                             transactionCache.addInserted(id, baoStrmToBytes);
546                         } else /*if (commit)*/ {
547                             throw writeError;
548                         }
549                     }
550                 }
551                 
552                 modStatus = cache.getModStatus();
553             }
554
555         if (classIndexChanged) {
556             writeError = replaceRecord(BtreeFactory.classIndexId, classIndex);
557                 transactionCache.addReplaced(BtreeFactory.classIndexId, baoStrmToBytes);
558         if (writeError != null)
559             throw writeError;
560         classIndexChanged = false;
561         }
562
563         // commit to disk
564 if (commit) {
565         fileCache.commit();
566                 transactionCache.clear ();
567         }
568     }
569     catch (StorageException ex) {
570         // Record that commit has failed.
571 saveFailed = true;
572         throw(ex);
573     } finally {
574             cache.updateSize();
575         }
576     }
577
578     /** roll back all changes
579     */
580     public void rollbackChanges() throws StorageException {
581         synchronized (myStorage) {
582             modificationLevel++;
583             Integer offset;
584             int dataOffset;
585
586             // iterator have to be obtained before closeFiles is called (it drops transitionCache)
587 TransactionCache.CacheIterator iter = transactionCache.iterator ();
588             long counter = dataFile.getMofIdCounter ();
589             boolean dataCommited = transactionCache.containsCommitedData ();
590
591             // throw away data in file cache
592 close();
593             open(false);
594
595             // redo operations commited to transaction cache
596 try {
597                 while (iter.hasNext ()) {
598                     TransactionCache.Record rec = iter.next ();
599                     switch (rec.op) {
600                         case TransactionCache.OP_DELETE:
601                             removeRecord(rec.id);
602                         break;
603                         case TransactionCache.OP_REPLACE:
604                             offset = (Integer )indexFile.get(rec.id);
605                             dataOffset = dataFile.replace(
606                                 offset.intValue(), rec.id.getSerialNumber(), rec.value);
607                             indexFile.replace(rec.id, new Integer (dataOffset));
608                         break;
609                         case TransactionCache.OP_INSERT:
610                             dataOffset = dataFile.put(rec.id.getSerialNumber(), rec.value);
611                             indexFile.add(rec.id, new Integer (dataOffset));
612                         break;
613                     } // switch
614
615                 } // while
616
617                 // reset mof id generator to the value before rollback
618 dataFile.setMofIdCounter (counter);
619                 fileCache.commit();
620             } catch (StorageException ex) {
621                 // Record that commit has failed.
622 saveFailed = true;
623                 throw(ex);
624             }
625         }
626     }
627
628     /** Deletes a repository record
629      * @param key -- a String
630      * @return true if record exists
631      */
632     public boolean remove(Object mKey) throws StorageException {
633         synchronized (myStorage) {
634             modificationLevel++;
635             if (!exists((MOFID)mKey)) {
636                 return false;
637             }
638             else {
639                 cache.remove(mKey);
640                 return true;
641             }
642         }
643     }
644             
645     /** Adds a repository record, throwing an exception if it already exists
646      * @param key -- a String
647      * @param value
648     */
649     public void add(Object key, Object value) throws StorageException {
650         MOFID mKey = (MOFID)key;
651     synchronized (myStorage) {
652             modificationLevel++;
653             if (exists(mKey)) {
654                 throw new StorageBadRequestException(
655                     MessageFormat.format("Record with key {0} already exists",
656                         new Object [] {mKey} ) );
657             }
658             addToCache(mKey, value);
659         }
660     }
661
662     /* Add object to cache */
663     private void addToCache(MOFID key, Object value) throws StorageException {
664         cache.put(key, value);
665         cache.setNew(key);
666     }
667
668     /** Replaces the original value associated with the specified key in this
669     * index with new value. If no value was associated with this key prior
670     * to this call StorageBadRequestException is thrown.
671     * @param key
672     * @param value
673     * @throws StorageException
674     */
675     public void replace(Object key, Object value) throws StorageException {
676         MOFID mKey = (MOFID)key;
677     synchronized (myStorage) {
678             modificationLevel++;
679             if (!exists(mKey)) {
680                 noSuchRecord(mKey);
681             }
682
683             replaceInCache(mKey, value);
684         }
685     }
686
687     /* Replace object in cache */
688     private void replaceInCache(MOFID key, Object value) throws StorageException {
689         boolean isNew = cache.isNew(key);
690         cache.replace(key, value);
691         if (isNew)
692             cache.setNew(key);
693         else
694             cache.setDirty(key);
695     }
696
697     /** Adds or replaces a repository record
698      * @param key -- a String
699      * @param value
700      * @return always null
701     */
702     public boolean put(Object key, Object value) throws StorageException {
703         synchronized (myStorage) {
704             modificationLevel++;
705             MOFID mKey = (MOFID)key;
706
707             if (!exists(mKey)) {
708                 addToCache(mKey, value);
709                 return false;
710             }
711             else {
712                 replaceInCache(mKey, value);
713                 return true;
714             }
715         }
716     }
717
718     /** Gets a record from the repository. Returns null if none exists.
719      * @param key -- a String
720      * @return value associated with specified key,
721      * or null if there was no mapping for key
722     */
723     public Object getIfExists(Object key)throws StorageException {
724     synchronized (myStorage) {
725             Object retval;
726             retval = cache.get(key);
727             if (retval == null) {
728                 if (cache.isDeleted(key)) {
729                     return null;
730                 }
731                 retval = getRecord((MOFID)key);
732                 if (retval != null) {
733                     cache.put(key, retval);
734                 }
735             }
736
737             return retval;
738         }
739     }
740
741     /** Like getIfExists, since we don't return keys */
742     public Object getObjectIfExists(Object key, SinglevaluedIndex dummy) throws StorageException {
743         return getIfExists(key);
744     }
745
746     /** Gets a record from the repository. Throws an exception if none exists.
747      * @param key
748      * @return value associated with specified key,
749      * or null if there was no mapping for key
750     */
751     public Object get(Object key)throws StorageException {
752         Object retval = getIfExists(key);
753         if (retval == null) {
754             noSuchRecord(key);
755         }
756
757         return retval;
758     }
759
760     /** Like get, since we don't return keys */
761     public Object getObject(Object key, SinglevaluedIndex dummy)
762         throws StorageException {
763         return get(key);
764     }
765
766     public Collection queryByKeyPrefix (Object prefix, SinglevaluedIndex repos) {
767         throw new UnsupportedOperationException ();
768     }
769     
770     /* Check if the record with the given key exists, either in the cache
771     * or in the files
772     */
773     private boolean exists(MOFID key) throws StorageException {
774         if (cache.get(key) != null) {
775             return true;
776         }
777         else if (cache.isDeleted(key)) {
778             return false;
779         }
780         else {
781             return indexFile.getIfExists(key) != null;
782         }
783     }
784
785     /* Removes the record associated in the index with specified key.
786      * or null if there was no mapping for key
787      */
788     private boolean removeRecord (MOFID mKey) throws StorageException {
789         Integer offset = (Integer )indexFile.getIfExists(mKey);
790         if (offset == null) {
791             noSuchRecord(mKey);
792         }
793
794         indexFile.remove(mKey);
795         dataFile.remove(offset.intValue(), mKey.getSerialNumber());
796         return true;
797     }
798
799     /** Mark that the object has changed, and so needs to be saved on commit
800     * @param key key of object whch changed (a String)
801     */
802     public void objectStateChanged(Object key) throws StorageException {
803     objectStateChanged((MOFID)key);
804     }
805
806     /** Mark that the object has changed, and so needs to be saved on commit
807     * @param mKey key of object whch changed
808     */
809     public void objectStateChanged(MOFID mKey) throws StorageException {
810         synchronized (myStorage) {
811             modificationLevel++;
812             cache.setDirty(mKey);
813         }
814     }
815
816     /** Fetch an index by name. Must be called from a block synchronized on myStorage.
817     * @param name name of index
818     */
819     Object fetchIndex(String name) throws StorageException {
820         fetchIndexIndex();
821         MOFID indexID = indexIndex.get(name);
822         return (indexID == null) ? null : get(indexID);
823     }
824
825     /** Drop an index by name. Must be called from a block synchronized on myStorage.
826     * @param name name of index
827     */
828     void dropIndex(String name) throws StorageException {
829         modificationLevel++;
830         fetchIndexIndex();
831         MOFID indexId = indexIndex.get(name);
832         indexIndex.remove(name);
833         objectStateChanged(BtreeFactory.indexIndexId);
834         remove(indexId);
835     }
836
837     /** List all index names
838     * @return an array of all the index names in alphabetical order
839     */
840     public String [] listIndexes() throws StorageException {
841         synchronized (myStorage) {
842             fetchIndexIndex();
843             String retval[] = indexIndex.listNames();
844             Arrays.sort(retval);
845             return retval;
846         }
847     }
848
849     /** Add a new index
850     * @param name name of index
851     * @param index the index object
852     * @param mID the MOFID of the index
853     */
854     void addIndex(String name, Object index, MOFID mID) throws StorageException {
855         modificationLevel++;
856         add(mID, index);
857         fetchIndexIndex();
858         indexIndex.add(name, mID);
859         objectStateChanged(BtreeFactory.indexIndexId);
860     }
861         
862     /* tell MOFIDs where they're being written */
863     private static MofidGenerator currentlyStreamingMofidGenerator = null;
864             
865     static MofidGenerator getCurrentlyStreamingMofidGenerator() {
866         return currentlyStreamingMofidGenerator;
867     }
868
869     /* serialize an object to output stream, return any exception */
870     private StorageException writeStreamable(Streamable obj) {
871     MofidGenerator previous = currentlyStreamingMofidGenerator;
872     currentlyStreamingMofidGenerator = dataFile;
873     try {
874         baoStrm.reset();
875             daoStrm.writeInt(getClassCode(obj.getClass()));
876             obj.write(daoStrm);
877             baoStrmToBytes = baoStrm.toByteArray();
878     } catch (StorageException ex) {
879         return ex;
880     }
881         catch (IOException ioException) {
882             return (StorageException) Logger.getDefault().annotate(new StorageIOException(ioException), ioException);
883         }
884     catch (RuntimeException th) {
885         return (StorageException) Logger.getDefault().annotate(new StorageTransientDataException(
886                 th.getClass().getName() + ": " + th.getMessage()), th);
887     }
888     finally {
889         currentlyStreamingMofidGenerator = previous;
890     }
891     // it worked
892 return null;
893     }
894     
895     private Streamable readStreamable (DataInputStream stream) throws StorageException {
896         int classCode;
897         Streamable data;
898         
899         try {
900             classCode = stream.readInt();
901         }
902         catch (IOException ex) {
903             throw new StorageIOException(ex);
904         }
905         try {
906             Class cls = (Class )classes.get(classCode);
907             data = (Streamable)cls.newInstance();
908         } catch (Exception ex) {
909             throw new StoragePersistentDataException(ex.getMessage());
910         }
911         if (data instanceof StorageClient) {
912             ((StorageClient)data).setStorage(myStorage);
913         }
914         data.read(stream);
915         return data;
916     }
917
918     /** write a record to the files.
919     * @return null if no error occurs.
920     * An exception if one occurs serializing the value
921     * @throws StorageException if any other error occurs
922     */
923     private StorageException addRecord(MOFID mKey, Object value) throws StorageException {
924         StorageException ex = writeStreamable((Streamable)value);
925     if (ex != null)
926         return ex;
927         int dataOffset = dataFile.put(mKey.getSerialNumber(), baoStrmToBytes);
928         indexFile.add(mKey, new Integer (dataOffset));
929     return null;
930     }
931             
932     /* get the code number for a class */
933     int getClassCode(Class cls) throws StorageException{
934     String className = cls.getName();
935         fetchClassIndex();
936     Integer i = classIndex.getIf(className);
937     if (i != null) {
938         return i.intValue();
939     }
940     else {
941         int code = classIndex.add(className);
942         classes.add(code, cls);
943         classIndexChanged = true;
944         return code;
945     }
946     }
947
948     /** replace a record in the files.
949     * @return null if no error occurs.
950     * An exception if one occurs serializing the value
951     * @throws StorageException if any other error occurs
952     */
953     private StorageException replaceRecord(MOFID mKey,Object value)throws StorageException {
954         StorageException ex = writeStreamable((Streamable)value);
955     if (ex != null)
956         return ex;
957         Integer offset = (Integer )indexFile.get(mKey);
958         int dataOffset = dataFile.replace(
959                     offset.intValue(), mKey.getSerialNumber(), baoStrmToBytes);
960         indexFile.replace(mKey, new Integer (dataOffset));
961     return null;
962     }
963             
964     /* reports that no such record exists */
965     private void noSuchRecord(Object key) throws StorageException {
966         throw new StorageBadRequestException(
967             MessageFormat.format("No record exists with key {0}",
968                 new Object [] {key} ) );
969     }
970
971     /* read a record from the files */
972     private Object getRecord(MOFID mKey) throws StorageException {
973         Integer offset = (Integer )indexFile.getIfExists(mKey);
974         if (offset == null) {
975             return null;
976         }
977
978     Streamable data;
979     InputStream strm = dataFile.get(offset.intValue(), mKey.getSerialNumber());
980     try {
981         DataInputStream inStream = new DataInputStream(strm);
982             data = this.readStreamable (inStream);
983     }
984     finally {
985         try {
986         strm.close();
987         }
988         catch (IOException ex) {
989         throw new StorageIOException(ex);
990         }
991     }
992         return data;
993     }
994
995     /* fetch the index index */
996     private void fetchIndexIndex() throws StorageException {
997         if (indexIndex == null) {
998             indexIndex = (MofidIndex)getIfExists(BtreeFactory.indexIndexId);
999             if (indexIndex == null) {
1000                indexIndex = new MofidIndex(this.myStorage);
1001                add(BtreeFactory.indexIndexId, indexIndex);
1002            }
1003        indexIndex.setName("Index of secondary Indexes");
1004        }
1005    }
1006                
1007    /* fetch the class index */
1008    private void fetchClassIndex() throws StorageException {
1009        if (classIndex == null) {
1010            classIndex = (CounterIndex)getIfExists(BtreeFactory.classIndexId);
1011            if (classIndex == null) {
1012                classIndex = new CounterIndex();
1013        // pre-load the class for the class index itself
1014 classIndex.add(CLASS_INDEX_TYPE);
1015                add(BtreeFactory.classIndexId, classIndex);
1016            }
1017        else {
1018            Iterator itr = classIndex.iterator();
1019        while (itr.hasNext()) {
1020            Map.Entry ent = (Map.Entry)itr.next();
1021            try {
1022            int code = ((Integer )ent.getValue()).intValue();
1023            while (classes.size() < code + 1) {
1024                classes.add(null);
1025            }
1026            classes.set(code, Class.forName((String )ent.getKey()));
1027            } catch (Exception ex) {
1028                throw new StoragePersistentDataException(
1029                            ex.getMessage());
1030            }
1031        }
1032        }
1033        classIndex.setName("Index of stored classes");
1034        }
1035    }
1036                
1037    /** Returns a set view of the keys contained in this index.
1038     * Returned collection is read only and may not be modified.
1039     * @return the set of all MOFIDs which are keys for the repository
1040     *
1041     */
1042    public java.util.Set keySet() throws StorageException {
1043        return new Keys();
1044    }
1045
1046    /** Returns a collection view of the values contained in the repository.
1047     * Returned collection is read only and may not be modified.
1048     * @return all objects stored in the repository
1049     */
1050    public java.util.Collection values () throws StorageException {
1051        return new Values();
1052    }
1053    
1054    /** Set our logging stream */
1055    public void setLoggingStream(PrintStream strm) {
1056        synchronized (myStorage) {
1057            loggingStream = strm;
1058        }
1059    }
1060
1061    /** Return the MOFID generator for this repository */
1062    public MofidGenerator getMofidGenerator() {
1063        synchronized (myStorage) {
1064            return dataFile;
1065        }
1066    }
1067
1068    private Map mofidMap = null;
1069
1070    /** Return the map of MOFID UUIDs we know about
1071    */
1072    public Map getMofidMap() {
1073        synchronized (myStorage) {
1074            // When we federae repositories, we'll need to know about all
1075 // of the federates. For now, there's only us.
1076 if (mofidMap == null) {
1077                mofidMap = new HashMap();
1078                mofidMap.put(dataFile.getMofidPrefix(), dataFile);
1079            }
1080            return mofidMap;
1081        }
1082    }
1083        
1084    /** Check consistency of btree database
1085    * @param strm where to write inconsistencies
1086    * @return number of errors encountered
1087    */
1088    public int checkConsistency(PrintWriter strm)
1089        throws StorageException {
1090        
1091        synchronized (myStorage) {
1092            /* Check consistency of data file */
1093            int numErrs =
1094                dataFile.dump(BtreeDataFile.DUMP_CONSISTENTCY_INFO, 0, false, strm);
1095
1096            /* Check that each key in the data file is properly in the index file */
1097            Iterator recordIter = dataFile.iterator(
1098                                    BtreeDataFile.ITERATE_NORMAL_EXTENTS);
1099
1100            while (recordIter.hasNext()) {
1101                NormalBtreeExtent ext = (NormalBtreeExtent)recordIter.next();
1102                MOFID mKey = this.myStorage.readMOFIDData (new ByteArrayInputStream (ext.key));
1103                Integer offset = (Integer )indexFile.getIfExists(mKey);
1104                if (offset == null) {
1105                    strm.println("ID " + mKey + " is not in the index file.");
1106                    numErrs++;
1107                } else if (offset.intValue() != ext.myChunkNum) {
1108                    strm.println("ID " + mKey + " has differring offsets: " +
1109                            ext.myChunkNum + " and " + offset.intValue());
1110                    numErrs++;
1111                }
1112            }
1113
1114            /* Check that cache is consistent with the files */
1115            Iterator actIter = cache.iterateActive();
1116            while (actIter.hasNext()) {
1117                MOFID key = (MOFID)actIter.next();
1118                if (cache.isDeleted(key)) {
1119                    if (!cache.isNew(key)) {
1120                        strm.println(
1121                            "ID " + key + " is deleted and active but not new");
1122                        numErrs++;
1123                    }
1124                }
1125                else {
1126                    if (cache.isNew(key)) {
1127                        if (inIndexFile(key)) {
1128                            strm.println(
1129                                "ID " + key + " is new but in the index file");
1130                            numErrs++;
1131                        }
1132                    }
1133                    else {
1134                        if (!inIndexFile(key)) {
1135                            strm.println(
1136                                "ID " + key +
1137                                " exists but is not in the index file");
1138                            numErrs++;
1139                        }
1140                    }
1141                }
1142            }
1143
1144            Iterator delIter = cache.iterateDeleted();
1145            while (delIter.hasNext()) {
1146                MOFID key = (MOFID)delIter.next();
1147                if (!inIndexFile(key)) {
1148                    strm.println(
1149                        "ID " + key + " is deleted but not in the index file");
1150                    numErrs++;
1151                }
1152            }
1153
1154            numErrs += ((Btree)indexFile).consistencyCheck(strm);
1155            strm.println("" + numErrs + " error(s) detected.");
1156            strm.println();
1157            cache.showStats(strm);
1158            strm.println();
1159            fileCache.showStats(strm);
1160            strm.println();
1161            strm.flush();
1162
1163            return numErrs;
1164        }
1165    }
1166    
1167        
1168    /* see if MOFID is in index file */
1169    private boolean inIndexFile(MOFID key) throws StorageException {
1170    return indexFile.getIfExists(key) != null;
1171    }
1172
1173    /** This will iterate over all keys in the repository
1174    */
1175    class KeyIterator implements Iterator {
1176
1177        /* the level of the database when we were created */
1178        private int iterModLevel;
1179
1180        /* iterate records in data file */
1181        private Iterator fileIter;
1182
1183        /* Iterate new objects in cache */
1184        private Iterator newIter;
1185
1186        /* next key to return */
1187        private MOFID nextKey;
1188    
1189        /** Create the iterator
1190    * @param internal if true, return MOFIDs instead of strings
1191    */
1192        KeyIterator(boolean internal) {
1193            fileIter = dataFile.iterator(BtreeDataFile.ITERATE_KEYS);
1194            newIter = null;
1195            iterModLevel = modificationLevel;
1196            getNextKey();
1197        }
1198
1199        /* Get the next key, first from the file, then from new
1200           objects in the cache */
1201        private void getNextKey() {
1202            nextKey = null;
1203
1204            synchronized(myStorage) {
1205                checkModLevel();
1206                while (fileIter.hasNext()) {
1207                    MOFID fileKey = (MOFID)fileIter.next();
1208                    if (!cache.isDeleted(fileKey)) {
1209                nextKey = fileKey;
1210                        return;
1211                    }
1212                }
1213                if (newIter == null) {
1214                    newIter = cache.iterateNew();
1215                }
1216
1217                if (newIter.hasNext()) {
1218                    nextKey = (MOFID)newIter.next();
1219                }
1220            }
1221        }
1222
1223        /** Is there another key? */
1224        public synchronized boolean hasNext() {
1225            return nextKey != null;
1226        }
1227
1228        /** Get the next key */
1229        public synchronized Object next() {
1230            Object current = nextKey;
1231            getNextKey();
1232            return current;
1233        }
1234
1235        /** Remove is unsupported */
1236        public void remove() {
1237            /* an optional operation which we do not support */
1238            throw new UnsupportedOperationException (
1239                "Remove is not supported");
1240        }
1241
1242        private void checkModLevel() {
1243            if (iterModLevel != modificationLevel) {
1244                throw new ConcurrentModificationException(
1245                    "Database had been modified");
1246            }
1247        }
1248    }
1249
1250    /** This will iterate over all objects in the repository
1251    */
1252    class ValueIterator implements Iterator {
1253        
1254        private Iterator keyIter;
1255        
1256        ValueIterator() {
1257            synchronized (myStorage) {
1258                keyIter = new KeyIterator(true);
1259            }
1260        }
1261
1262        /** returns true if there is another value to iterate over
1263        * @return true if there is another value to iterate over
1264        */
1265        public boolean hasNext() {
1266            return keyIter.hasNext();
1267        }
1268
1269        /** Returns the next value
1270        * @return the next value
1271        */
1272        public Object next() {
1273            MOFID id = (MOFID)keyIter.next();
1274            try {
1275                return get(id);
1276            }
1277            catch (StorageException ex) {
1278                throw new RuntimeStorageException(ex);
1279            }
1280        }
1281        
1282        /** Remove is not suppported
1283        * @exception UnsupportedOperationException always thrown
1284        */
1285        public void remove() {
1286            /* an optional operation which we do not support */
1287            throw new UnsupportedOperationException (
1288                "Remove is not supported");
1289        }
1290    }
1291
1292
1293    /**
1294    * This is the object returned by BtreeDatabase.keySet(). It uses
1295    * the iterator classes to implement the Set interface.
1296    */
1297    class Keys extends AbstractSet implements Set {
1298        
1299        /** return an iterator over all data in the repository
1300        * @return iterator
1301        */
1302        public Iterator iterator() {
1303            synchronized (myStorage) {
1304                return new KeyIterator(false);
1305            }
1306        }
1307
1308        /** returns number of objects in repository
1309        * @return number of objects in repository
1310        */
1311        public int size() {
1312            return BtreeDatabase.this.size();
1313        }
1314    }
1315
1316    /**
1317    * This is the object returned by BtreeDatabase.values(). It uses
1318    * the iterator classes to implement the Collection interface. Note
1319    * that since we don't index on the objects stored in the repository,
1320    * only on their MOFIDs, operations like contains will do linear
1321    * searches, instantiating objects as they go.
1322    */
1323    class Values extends AbstractCollection implements Collection {
1324        
1325        /** return an iterator over all data in the repository
1326        * @return iterator
1327        */
1328        public Iterator iterator() {
1329            return BtreeDatabase.this.new ValueIterator();
1330        }
1331
1332        /** returns number of objects in repository
1333        * @return number of objects in repository
1334        */
1335        public int size() {
1336            return BtreeDatabase.this.size();
1337        }
1338    }
1339    
1340}
1341

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags