KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > com > jofti > store > AbstractStoreManager


1 package com.jofti.store;
2
3 import java.io.IOException JavaDoc;
4 import java.nio.ByteBuffer JavaDoc;
5 import java.util.Arrays JavaDoc;
6 import java.util.Iterator JavaDoc;
7 import java.util.Properties JavaDoc;
8
9 import org.apache.commons.logging.Log;
10 import org.apache.commons.logging.LogFactory;
11
12 import com.jofti.btree.BTree;
13 import com.jofti.btree.EntrySplitWrapper;
14 import com.jofti.btree.IPage;
15 import com.jofti.btree.LeafNodeEntry;
16 import com.jofti.core.IStoreKey;
17 import com.jofti.core.IStoreManager;
18 import com.jofti.exception.JoftiException;
19
20 import edu.emory.mathcs.backport.java.util.concurrent.atomic.AtomicLong;
21
22
23 /**
24  * <p>
25  * Base store manager that deals with non-implementation specific actions. The majority of the
26  * functions are to do with storing and retrieving pages to and from disk.
27  * </p>
28  * <p>
29  * Sub classesshould make use of the bstract methods provided to allow diffirent types of
30  * cache optimisations to make alleviate some of the direct to disk behaviour.
31  * </p>
32  * @author xenephon
33  *
34  */

35 public abstract class AbstractStoreManager implements IStoreManager, IEntrySerializer {
36
37
38
39     // values used for initial properties
40
final String JavaDoc OVERFLOW_DIRECTORY ="directory";
41     final String JavaDoc BLOCK_SIZE = "block-size";
42     final String JavaDoc FILE_SIZE = "file-size";
43     final String JavaDoc MAX_NODES = "max-nodes";
44     final String JavaDoc MAX_FILES ="max-files";
45     final String JavaDoc FILE_NAME ="filename";
46     
47     
48     /* objects that manage the reusable objects */
49     
50     //manages the file access and writing
51
FileManager logManager = null;
52     
53     // buffers used to transfer data to the files
54
BufferManager bufferManager =null;
55     
56     // serialization helpers
57
HelperManager helperManager =null;
58     
59     //manager for pages
60
PageManager pageManager =null;
61     
62     //marker for curent storepointer
63
protected int currentStore =0;
64     
65     // used to issue new keys per node
66
final AtomicLong keyId =new AtomicLong(0);
67    
68     // properties
69
protected Properties JavaDoc configProperties =null;
70     protected String JavaDoc name =null;
71     
72     private static Log log = LogFactory.getLog(AbstractStoreManager.class);
73     
74      
75     int misses=0;
76     int hits =0;
77     int pendingHits =0;
78     int storeRetrieves=0;
79     
80     int size =0;
81
82
83     
84     int blockSize =4096; //default is 4k
85
int fileSize=30485760; // default is 300Mb
86
int maxFiles =10; //default is 10
87
String JavaDoc overflowDirectory ="/tmp"; //temp directory for operating system
88
String JavaDoc fileName ="jofti";
89     
90     
91     int maxBuffers=100; //maximum reusable buffer objects - depends on threads
92
int bufferNumber=10; // number of buffers to create initially
93

94      int maxNodes =3000; //default max nodes to keep in memory
95
int helperNumber =30; // number of helpers - should be able to cope with maximum number of threads
96

97
98  
99
100      
101     /* (non-Javadoc)
102      * @see com.jofti.store.IStoreManager#init(java.util.Properties)
103      */

104      public synchronized void init(Properties JavaDoc properties) throws JoftiException{
105
106
107          setUpProperties(properties);
108
109          if (log.isInfoEnabled()){
110                 log.info("block size is " + blockSize);
111                 log.info("max file length is " + fileSize);
112                 log.info("max nodes are " + maxNodes);
113                 log.info("max file stores is " + maxFiles);
114                 log.info("Overflow directory is " + overflowDirectory);
115                 log.info("filename is " + fileName);
116
117             }
118          //set up the log files
119
logManager = new FileManager();
120          logManager.init(blockSize,overflowDirectory,fileName,fileSize,maxFiles);
121          
122          //set up the buffers that transfer data to the files
123
bufferManager = new BufferManager();
124          bufferManager.init(blockSize,maxBuffers,bufferNumber);
125                        
126          //set up a serialization helper
127
helperManager = new HelperManager();
128          helperManager.init(helperNumber,blockSize,BTree.getMaxNodeSize(),this);
129     
130          //set up the page manager
131
pageManager = new PageManager();
132          pageManager.init(blockSize,100,10,this,this);
133          
134  
135         }
136     
137
138    
139     /**
140      * @param properties
141      * @throws JoftiException
142      */

143     protected void setUpProperties(Properties JavaDoc properties) throws JoftiException
144     {
145         if (properties != null){
146             configProperties = properties;
147             String JavaDoc key = null;
148             for (Iterator JavaDoc it = properties.keySet().iterator();it.hasNext();){
149                 key = (String JavaDoc)it.next();
150                 if (BLOCK_SIZE.equalsIgnoreCase(key)){
151                     String JavaDoc tempSize =properties.getProperty(key);
152                     try {
153                         blockSize =Integer.parseInt(tempSize);
154                     }catch (Exception JavaDoc e){
155                         throw new JoftiException("block-size of " + tempSize +" is not valid");
156                     }
157                 }
158                 
159                 if (FILE_SIZE.equalsIgnoreCase(key)){
160                     String JavaDoc tempFile =properties.getProperty(key);
161                     try {
162                         fileSize =Integer.parseInt(tempFile);
163                     }catch (Exception JavaDoc e){
164                         throw new JoftiException("file lengths of " + tempFile +" is not valid");
165                     }
166                 }
167                 if (MAX_NODES.equalsIgnoreCase(key)){
168                     String JavaDoc temp =properties.getProperty(key);
169                     try {
170                         maxNodes =Integer.parseInt(temp);
171                     }catch (Exception JavaDoc e){
172                         throw new JoftiException("max nodes of " + temp +" is not valid");
173                     }
174                 }
175             
176                 if (MAX_FILES.equalsIgnoreCase(key)){
177                     String JavaDoc temp =properties.getProperty(key);
178                     try {
179                         maxFiles =Integer.parseInt(temp);
180                     }catch (Exception JavaDoc e){
181                         throw new JoftiException("max store number of " + temp +" is not valid");
182                     }
183
184                 }
185                  if (OVERFLOW_DIRECTORY.equalsIgnoreCase(key)){
186
187                      overflowDirectory = properties.getProperty(key);
188
189                  }
190                  if (FILE_NAME.equalsIgnoreCase(key)){
191
192                     fileName = properties.getProperty(key);
193
194
195
196                 }
197                 
198             }
199         }
200     }
201     
202   
203    
204  
205     /* (non-Javadoc)
206      * @see com.jofti.store.IStoreManager#getName()
207      */

208     public String JavaDoc getName() {
209         return name;
210     }
211     /* (non-Javadoc)
212      * @see com.jofti.store.IStoreManager#setName(java.lang.String)
213      */

214     public void setName(String JavaDoc name) {
215         this.name = name;
216     }
217
218     /* (non-Javadoc)
219      * @see com.jofti.core.IStoreManager#getNextKey()
220      */

221     public IStoreKey getNextKey(){
222         StoreKey key = new StoreKey(keyId.getAndIncrement());
223         try {
224             key.setFilePositions(new FilePositionHolder[]{logManager.getNextPosition()});
225         } catch(JoftiException e){
226             throw new RuntimeException JavaDoc(e);
227         }
228         key.newKey=true;
229         return key;
230     }
231
232     
233         
234     /* (non-Javadoc)
235      * @see com.jofti.core.IStoreManager#store(com.jofti.core.IStoreKey, com.jofti.btree.IPage)
236      */

237     public abstract IStoreKey store(IStoreKey key, IPage obj) throws JoftiException;
238        
239     /* (non-Javadoc)
240      * @see com.jofti.core.IStoreManager#retrieve(com.jofti.core.IStoreKey)
241      */

242     public abstract StoreWrapper retrieve(IStoreKey key) throws JoftiException;
243   
244     
245     protected FilePositionHolder[] allocatePositions(FilePositionHolder[] filePositions, int size) throws JoftiException{
246         return logManager.allocateBlocks(filePositions,size);
247         
248     }
249
250
251     /**
252      * <p>
253      * Method that does the actual work for storing a byteBuffer to disk. The Byte buffer can be
254      * of any size and the filepositions in the key object are used to determine where the data is written.
255      * </p>
256      * @param key
257      * @param buffer
258      * @return
259      * @throws JoftiException
260      */

261     protected IStoreKey doStore(IStoreKey key, ByteBuffer JavaDoc buffer)
262             throws JoftiException
263     {
264
265         buffer.rewind();
266         FilePositionHolder[] positions = key.getFilePositions();
267
268         if (positions.length == 1) {
269             BlockBuffer buf = (BlockBuffer) bufferManager
270                     .acquireBuffer(positions[0]);
271             // write a single buffer
272
try {
273                 buf.writeFurniture(key.getNumber());
274                 buf.put(buffer);
275                 FileStore file = logManager.getFileStore(positions[0].file);
276                 buf.file = file;
277                 buf.write();
278             } catch (IOException JavaDoc ie) {
279                 throw new JoftiException(ie);
280             } finally {
281                 bufferManager.releaseBuffer(buf);
282             }
283         } else {
284             // we need to loop through the positions and write out the buffers
285
// as we have a node that is too big for single buffer
286
int offSet = 0;
287             int originalLimit = buffer.limit();
288             
289             for (int i = 0; i < positions.length; i++) {
290                 BlockBuffer buf = (BlockBuffer) bufferManager
291                         .acquireBuffer(positions[i]);
292                 try {
293                     buf.writeFurniture(key.getNumber());
294
295                     // set the pos to be the first offset
296
buffer.position(offSet);
297                     // set a temp limit
298
if (originalLimit - offSet > buf.remaining()) {
299                         buffer.limit(offSet + buf.remaining());
300                         offSet = buffer.limit();
301                     } else {
302                         buffer.limit(originalLimit);
303                         offSet = originalLimit;
304                     }
305
306                     buf.put(buffer);
307
308                     buf.file = logManager.getFileStore(positions[i].file);
309                     buf.write();
310                 } catch (IOException JavaDoc ie) {
311                     throw new JoftiException(ie);
312                 } finally {
313                     bufferManager.releaseBuffer(buf);
314                 }
315
316             }
317
318         }
319
320         return key;
321     }
322     
323      /* (non-Javadoc)
324      * @see com.jofti.core.IStoreManager#releasePage(com.jofti.core.IStoreKey, com.jofti.btree.IPage)
325      */

326     public abstract void releasePage(IStoreKey key, IPage page);
327    
328
329     /**
330      * <p>
331      * Used to retrieve a new page. Sub classes should use this method to decorate the
332      * pages in any way they want in order to improve any cache optimisations.
333      * </p>
334      * @param size
335      * @return
336      */

337     protected abstract IPage getNewPage(int size);
338    
339     /**
340      * <p>
341      * The method that does the work of retrieving the page from the backing store.
342      * </p>
343      * @param key
344      * @return
345      * @throws JoftiException
346      */

347
348     protected IPage doRetrieve(IStoreKey key) throws JoftiException
349     {
350
351         if (key.isNewKey()) {
352             return doGetNewPage(blockSize);
353         }
354
355         IPage page = null;
356
357         try {
358             FilePositionHolder[] positions = key.getFilePositions();
359
360             if (positions.length == 1) {
361                 // just get the values straight
362
BlockBuffer buf = bufferManager.acquireBuffer(positions[0]);
363                 ExternalisableHelper helper = helperManager.acquireHelper();
364                 try {
365                     FileStore file = logManager.getFileStore(positions[0].file);
366                     buf.read(file);
367
368                     page = helper.readExternalBuffer(buf.buffer, key
369                             .getNumber());
370                     page.setManager(this);
371                     return page;
372                 } finally {
373                     bufferManager.releaseBuffer(buf);
374                     helperManager.releaseHelper(helper);
375                 }
376             } else {
377                 // merge all the buffers into a single buffer
378
ByteBuffer JavaDoc totalBuf = pageManager
379                         .acquireBuffer(positions.length * blockSize);
380                 totalBuf.clear();
381                 ExternalisableHelper helper = helperManager.acquireHelper();
382                 // Object[] obj =null;
383
int number = 0;
384                 try {
385                     for (int i = 0; i < positions.length; i++) {
386                         BlockBuffer buf = bufferManager
387                                 .acquireBuffer(positions[i]);
388
389                         try {
390                             FileStore file = logManager
391                                     .getFileStore(positions[i].file);
392                             buf.read(file);
393                             totalBuf.put(buf.buffer);
394                             if (i == 0) {
395                                 number = buf.numberEntries;
396                             }
397                         } finally {
398                             bufferManager.releaseBuffer(buf);
399                         }
400                     }
401                     //
402
totalBuf.flip();
403                     try {
404                         page = helper.readExternalBuffer(totalBuf, number);
405                     } catch (Throwable JavaDoc t) {
406                         log.fatal("buffer read failure ", t);
407                         throw t;
408                     }
409                     page.setManager(this);
410                 } finally {
411                     helperManager.releaseHelper(helper);
412                     pageManager.releaseBuffer(totalBuf);
413                 }
414                 return page;
415             }
416         } catch (Throwable JavaDoc t) {
417             log.fatal("error retrieving record ", t);
418             throw new JoftiException("error retrieving record ", t);
419         }
420
421     }
422
423     /* (non-Javadoc)
424      * @see com.jofti.core.IStoreManager#remove(com.jofti.core.IStoreKey, com.jofti.btree.IPage)
425      */

426     public abstract void remove(IStoreKey key, IPage page) throws JoftiException ;
427     
428     
429     /* (non-Javadoc)
430      * @see com.jofti.core.IStoreManager#removeAll()
431      */

432     public abstract void removeAll() throws JoftiException;
433     
434     
435     protected void doRemoveAll() throws JoftiException{
436         
437         logManager.reset();
438     }
439     
440     protected void doRemove(IStoreKey key) throws JoftiException {
441         
442             FilePositionHolder[] holder = key.getFilePositions();
443         
444             if (holder != null){
445                 for (int i=0;i<holder.length;i++){
446                     logManager.removePosition(holder[i]);
447                 }
448                 
449             }
450
451         
452     }
453     
454     /* Serializer methods */
455
456     
457     /* (non-Javadoc)
458      * @see com.jofti.store.IEntrySerializer#convertForStorage(java.lang.Object)
459      */

460     public byte[] convertForStorage(Object JavaDoc obj) throws JoftiException{
461             ExternalisableHelper helper =helperManager.acquireHelper();
462             try {
463                 return helper.convertForStore((LeafNodeEntry)obj);
464             }finally {
465                 helperManager.releaseHelper(helper);
466             }
467     }
468     
469     
470     
471  
472     /* (non-Javadoc)
473      * @see com.jofti.store.IEntrySerializer#convertFromStorage(java.nio.ByteBuffer)
474      */

475     public LeafNodeEntry convertFromStorage(ByteBuffer JavaDoc buf) throws JoftiException{
476             ExternalisableHelper helper =helperManager.acquireHelper();
477             try {
478                 return (LeafNodeEntry) helper.convertFromStore(buf);
479             }finally {
480                 helperManager.releaseHelper(helper);
481             }
482     }
483     
484     /**
485      * @param size
486      * @return
487      */

488     protected IPage doGetNewPage(int size)
489     {
490
491         IPage page = null;
492
493         try {
494             page = pageManager.acquirePage(size);
495         } catch (Exception JavaDoc e) {
496             throw new RuntimeException JavaDoc("unable to acquire page for size "
497                     + size);
498         }
499         return page;
500
501     }
502     
503   
504     /* (non-Javadoc)
505      * @see com.jofti.core.IStoreManager#split(com.jofti.btree.IPage, int)
506      */

507     public EntrySplitWrapper[] split(IPage page, int entryNumber){
508         
509
510         int splitPoint = entryNumber/2;
511             
512         //set up the new buffer
513
ByteBuffer JavaDoc buf = page.getBuffer();
514         int[] pointers =page.getPointers();
515         // see if we need more than block size
516
int endPoint = pointers[entryNumber-1];
517
518          buf.position(endPoint);
519         int endLength = buf.getInt()+4;
520         
521         int bufferSize =(endLength+endPoint) - pointers[splitPoint];
522         
523         IPage newPage =getNewPage(bufferSize);
524         
525
526     
527         newPage.getBuffer().limit(bufferSize);
528         newPage.getBuffer().rewind();
529         
530         int length = entryNumber -splitPoint;
531         System.arraycopy(pointers,splitPoint,newPage.getPointers(),0,length);
532         
533         
534         // write the buffer
535
buf.position(newPage.getPointers()[0]);
536         
537         buf.mark();
538         try {
539             newPage.getBuffer().put(buf);
540         } catch (Throwable JavaDoc t){
541             log.fatal("error "+ newPage.getBuffer() + " "+ buf);
542             throw new RuntimeException JavaDoc(t);
543         }
544         newPage.getBuffer().flip();
545
546         buf.reset();
547         buf.flip();
548         
549
550         
551         
552 // erase the higher entries
553
Arrays.fill(pointers,splitPoint,pointers.length-1,-1);
554     
555         // ok nor reset the higher fields
556
int offSet = newPage.getPointers()[0];
557         
558         for (int i=0;i<length;i++){
559             newPage.getPointers()[i]= newPage.getPointers()[i]-offSet;
560         }
561         
562         EntrySplitWrapper[] entries =new EntrySplitWrapper[2];
563         EntrySplitWrapper splitWrapper = new EntrySplitWrapper();
564         splitWrapper.entries =page;
565         splitWrapper.size=length;
566         entries[0] = splitWrapper;
567         
568         
569         EntrySplitWrapper newWrapper = new EntrySplitWrapper();
570         newWrapper.entries =newPage;
571         newWrapper.size=splitPoint;
572         
573         entries[1]=newWrapper;
574         
575         return entries;
576         
577     }
578   
579
580
581 }
582
Popular Tags