Java > Open Source Codes > org > archive > util > CachedBdbMap


1 /* CachedBdbMap
2  *
3  * $Id: CachedBdbMap.java,v 1.25.4.1 2007/01/13 01:31:39 stack-sf Exp $
4  *
5  * Created on Mar 24, 2004
6  *
7  * Copyright (C) 2004 Internet Archive.
8  *
9  * This file is part of the Heritrix web crawler (crawler.archive.org).
10  *
11  * Heritrix is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU Lesser Public License as published by
13  * the Free Software Foundation; either version 2.1 of the License, or
14  * any later version.
15  *
16  * Heritrix is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19  * GNU Lesser Public License for more details.
20  *
21  * You should have received a copy of the GNU Lesser Public License
22  * along with Heritrix; if not, write to the Free Software
23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24  */
25 package org.archive.util;
26
27 import java.io.File ;
28 import java.io.IOException ;
29 import java.io.Serializable ;
30 import java.lang.ref.PhantomReference ;
31 import java.lang.ref.Reference ;
32 import java.lang.ref.ReferenceQueue ;
33 import java.lang.ref.SoftReference ;
34 import java.lang.reflect.Field ;
35 import java.util.AbstractMap ;
36 import java.util.HashMap ;
37 import java.util.Iterator ;
38 import java.util.LinkedList ;
39 import java.util.Map ;
40 import java.util.Set ;
41 import java.util.logging.Level ;
42 import java.util.logging.Logger ;
43
44 import com.sleepycat.bind.EntryBinding;
45 import com.sleepycat.bind.serial.SerialBinding;
46 import com.sleepycat.bind.serial.StoredClassCatalog;
47 import com.sleepycat.bind.tuple.TupleBinding;
48 import com.sleepycat.collections.StoredSortedMap;
49 import com.sleepycat.je.Database;
50 import com.sleepycat.je.DatabaseConfig;
51 import com.sleepycat.je.DatabaseException;
52 import com.sleepycat.je.Environment;
53 import com.sleepycat.je.EnvironmentConfig;
54
55 /**
56  * A BDB JE backed hashmap. It extends the normal BDB JE map implementation by
57  * holding a cache of soft referenced objects. That is objects are not written
58  * to disk until they are not referenced by any other object and therefore can be
59  * Garbage Collected.
60  *
61  * @author John Erik Halse
62  * @author stack
63  * @author gojomo
64  *
65  */
66 public class CachedBdbMap<K,V> extends AbstractMap <K,V>
67 implements Map <K,V>, Serializable {
68     
69     private static final long serialVersionUID = -8655539411367047332L;
70
71     private static final Logger logger =
72         Logger.getLogger(CachedBdbMap.class.getName());
73
74     /** The database name of the class definition catalog.*/
75     private static final String CLASS_CATALOG = "java_class_catalog";
76
77     /**
78      * A map of BDB JE Environments so that we reuse the Environment for
79      * databases in the same directory.
80      */
81     private static final Map <String ,DbEnvironmentEntry> dbEnvironmentMap =
82         new HashMap <String ,DbEnvironmentEntry>();
83
84     /** The BDB JE environment used for this instance.
85      */
86     private transient DbEnvironmentEntry dbEnvironment;
87
88     /** The BDB JE database used for this instance. */
89     protected transient Database db;
90
91     /** The Collection view of the BDB JE database used for this instance. */
92     protected transient StoredSortedMap diskMap;
93
94     /** The softreferenced cache */
95     private transient Map <K,SoftEntry<V>> memMap;
96
97     protected transient ReferenceQueue <V> refQueue;
98
99     /** The number of objects in the diskMap StoredMap.
100      * (Package access for unit testing.) */
101     protected int diskMapSize = 0;
102
103     /**
104      * Count of times we got an object from in-memory cache.
105      */
106     private long cacheHit = 0;
107
108     /**
109      * Count of times the {@link CachedBdbMap#get(Object)} method was called.
110      */
111     private long countOfGets = 0;
112
113     /**
114      * Count of every time we went to the disk-based map AND we found an
115      * object (Doesn't include accesses that came back null).
116      */
117     private long diskHit = 0;
118     
119     /**
120      * Name of bdbje db.
121      */
122     private String dbName = null;
123
124     /**
125      * Reference to the Reference#referent Field.
126      */
127     protected static Field referentField;
128     static {
129         // We need access to the referent field in the PhantomReference.
130 // For more on this trick, see
131 // http://www.javaspecialists.co.za/archive/Issue098.html and for
132 // discussion:
133 // http://www.theserverside.com/tss?service=direct/0/NewsThread/threadViewer.markNoisy.link&sp=l29865&sp=l146901
134 try {
135             referentField = Reference .class.getDeclaredField("referent");
136             referentField.setAccessible(true);
137         } catch (SecurityException e) {
138             throw new RuntimeException (e);
139         } catch (NoSuchFieldException e) {
140             throw new RuntimeException (e);
141         }
142     }
143
144     /**
145      * Simple structure to keep needed information about a DB Environment.
146      */
147     protected static class DbEnvironmentEntry {
148         Environment environment;
149         StoredClassCatalog classCatalog;
150         int openDbCount = 0;
151         File dbDir;
152     }
153     
154     /**
155      * Shudown default constructor.
156      */
157     private CachedBdbMap() {
158         super();
159     }
160     
161     /**
162      * Constructor.
163      *
164      * You must call
165      * {@link #initialize(Environment, Class, Class, StoredClassCatalog)}
166      * to finish construction. Construction is two-stepped to support
167      * reconnecting a deserialized CachedBdbMap with its backing bdbje
168      * database.
169      *
170      * @param dbName Name of the backing db this instance should use.
171      */
172     public CachedBdbMap(final String dbName) {
173         this();
174         this.dbName = dbName;
175     }
176
177     /**
178      * A constructor for creating a new CachedBdbMap.
179      *
180      * Even though the put and get methods conforms to the Collections interface
181      * taking any object as key or value, you have to submit the class of the
182      * allowed key and value objects here and will get an exception if you try
183      * to put anything else in the map.
184      *
185      * <p>This constructor internally calls
186      * {@link #initialize(Environment, Class, Class, StoredClassCatalog)}.
187      * Do not call initialize if you use this constructor.
188      *
189      * @param dbDir The directory where the database will be created.
190      * @param dbName The name of the database to back this map by.
191      * @param keyClass The class of the objects allowed as keys.
192      * @param valueClass The class of the objects allowed as values.
193      *
194      * @throws DatabaseException is thrown if the underlying BDB JE database
195      * throws an exception.
196      */
197     public CachedBdbMap(final File dbDir, final String dbName,
198             final Class <K> keyClass, final Class <V> valueClass)
199     throws DatabaseException {
200         this(dbName);
201         this.dbEnvironment = getDbEnvironment(dbDir);
202         this.dbEnvironment.openDbCount++;
203         initialize(dbEnvironment.environment, keyClass, valueClass,
204             dbEnvironment.classCatalog);
205         if (logger.isLoggable(Level.INFO)) {
206             // Write out the bdb configuration.
207 EnvironmentConfig cfg = this.dbEnvironment.environment.getConfig();
208             logger.info("BdbConfiguration: Cache percentage " +
209                 cfg.getCachePercent() + ", cache size " + cfg.getCacheSize() +
210                 ", Map size: " + size());
211         }
212     }
213     
214     /**
215      * Call this method when you have an instance when you used the
216      * default constructor or when you have a deserialized instance that you
217      * want to reconnect with an extant bdbje environment. Do not
218      * call this method if you used the
219      * {@link #CachedBdbMap(File, String, Class, Class)} constructor.
220      * @param env
221      * @param keyClass
222      * @param valueClass
223      * @param classCatalog
224      * @throws DatabaseException
225      */
226     public synchronized void initialize(final Environment env, final Class keyClass,
227             final Class valueClass, final StoredClassCatalog classCatalog)
228     throws DatabaseException {
229         initializeInstance();
230         this.db = openDatabase(env, this.dbName);
231         this.diskMap = createDiskMap(this.db, classCatalog, keyClass,
232             valueClass);
233     }
234     
235     /**
236      * Do any instance setup.
237      * This method is used by constructors and when deserializing an instance.
238      */
239     protected void initializeInstance() {
240         this.memMap = new HashMap <K,SoftEntry<V>>();
241         this.refQueue = new ReferenceQueue <V>();
242     }
243     
244     protected StoredSortedMap createDiskMap(Database database,
245             StoredClassCatalog classCatalog, Class keyClass, Class valueClass) {
246         EntryBinding keyBinding = TupleBinding.getPrimitiveBinding(keyClass);
247         if(keyBinding == null) {
248             keyBinding = new SerialBinding(classCatalog, keyClass);
249         }
250         EntryBinding valueBinding = TupleBinding.getPrimitiveBinding(valueClass);
251         if(valueBinding == null) {
252             valueBinding = new SerialBinding(classCatalog, valueClass);
253         }
254         return new StoredSortedMap(database, keyBinding, valueBinding, true);
255     }
256
257     /**
258      * Get the database environment for a physical directory where data will be
259      * stored.
260      * <p>
261      * If the environment already exist it will be reused, else a new one will
262      * be created.
263      *
264      * @param dbDir The directory where BDB JE data will be stored.
265      * @return a datastructure containing the environment and a default database
266      * for storing class definitions.
267      */
268     private DbEnvironmentEntry getDbEnvironment(File dbDir) {
269         if (dbEnvironmentMap.containsKey(dbDir.getAbsolutePath())) {
270             return (DbEnvironmentEntry) dbEnvironmentMap.get(dbDir
271                     .getAbsolutePath());
272         }
273         EnvironmentConfig envConfig = new EnvironmentConfig();
274         envConfig.setAllowCreate(true);
275         envConfig.setTransactional(false);
276         
277         // We're doing the caching ourselves so setting these at the lowest
278 // possible level.
279 envConfig.setCachePercent(1);
280         DbEnvironmentEntry env = new DbEnvironmentEntry();
281         try {
282             env.environment = new Environment(dbDir, envConfig);
283             env.dbDir = dbDir;
284             dbEnvironmentMap.put(dbDir.getAbsolutePath(), env);
285             
286             DatabaseConfig dbConfig = new DatabaseConfig();
287             dbConfig.setTransactional(false);
288             dbConfig.setAllowCreate(true);
289             
290             Database catalogDb = env.environment.openDatabase(null,
291                     CLASS_CATALOG, dbConfig);
292             
293             env.classCatalog = new StoredClassCatalog(catalogDb);
294         } catch (DatabaseException e) {
295             e.printStackTrace();
296             //throw new FatalConfigurationException(e.getMessage());
297 }
298         return env;
299     }
300
301     protected Database openDatabase(final Environment environment,
302             final String dbName) throws DatabaseException {
303         DatabaseConfig dbConfig = new DatabaseConfig();
304         dbConfig.setTransactional(false);
305         dbConfig.setAllowCreate(true);
306         return environment.openDatabase(null, dbName, dbConfig);
307     }
308
309     public synchronized void close() throws DatabaseException {
310         // Close out my bdb db.
311 if (this.db != null) {
312             try {
313                 this.db.close();
314             } catch (DatabaseException e) {
315                 e.printStackTrace();
316             } finally {
317                 this.db = null;
318             }
319         }
320         if (dbEnvironment != null) {
321             dbEnvironment.openDbCount--;
322             if (dbEnvironment.openDbCount <= 0) {
323                 dbEnvironment.classCatalog.close();
324                 dbEnvironment.environment.close();
325                 dbEnvironmentMap.remove(dbEnvironment.dbDir.getAbsolutePath());
326                 dbEnvironment = null;
327             }
328         }
329     }
330
331     protected void finalize() throws Throwable {
332         close();
333         super.finalize();
334     }
335
336     /**
337      * The keySet of the diskMap is all relevant keys.
338      *
339      * @see java.util.Map#keySet()
340      */
341     @SuppressWarnings ("unchecked")
342     public Set <K> keySet() {
343         return diskMap.keySet();
344     }
345     
346     public Set <Map.Entry <K,V>> entrySet() {
347         // Would require complicated implementation to
348 // maintain identity guarantees, so skipping
349 throw new UnsupportedOperationException ();
350     }
351
352     public synchronized V get(final Object object) {
353         K key = toKey(object);
354         countOfGets++;
355         expungeStaleEntries();
356         if (countOfGets % 10000 == 0) {
357             logCacheSummary();
358         }
359         SoftEntry<V> entry = memMap.get(key);
360         if (entry != null) {
361             V val = entry.get(); // get & hold, so not cleared pre-return
362 if (val != null) {
363                 cacheHit++;
364                 return val;
365             }
366             // Explicitly clear this entry from referencequeue since its
367 // value is null.
368 expungeStaleEntry(entry);
369         }
370
371         // check backing diskMap
372 V v = diskMapGet(key);
373         if (v != null) {
374             diskHit++;
375             memMap.put(key, new SoftEntry<V>(key, v, refQueue));
376         }
377         return v;
378     }
379
380     /**
381      * Info to log, if at FINE level, on every get()
382      */
383     private void logCacheSummary() {
384         if (!logger.isLoggable((Level.FINE))) {
385             return;
386         }
387         try {
388             long cacheHitPercent = (cacheHit * 100) / (cacheHit + diskHit);
389             logger.fine("DB name: " + this.db.getDatabaseName()
390                 + ", Cache Hit: " + cacheHitPercent
391                 + "%, Not in map: " + (countOfGets - (cacheHit + diskHit))
392                 + ", Total number of gets: " + countOfGets);
393         } catch (DatabaseException e) {
394             // This is just for logging so ignore DB Exceptions
395 }
396     }
397     
398     public synchronized V put(K key, V value) {
399         V prevVal = get(key);
400         memMap.put(key, new SoftEntry<V>(key, value, refQueue));
401         diskMap.put(key,value); // dummy
402 if(prevVal==null) {
403             diskMapSize++;
404         }
405         return prevVal;
406     }
407
408     /**
409      * Note that a call to this method CLOSEs the underlying bdbje.
410      * This instance is no longer of any use. It must be re-initialized.
411      * We close the db here because if this BigMap is being treated as a plain
412      * Map, this is only opportunity for cleanup.
413      */
414     public synchronized void clear() {
415         this.memMap.clear();
416         this.diskMap.clear();
417         this.diskMapSize = 0;
418         try {
419             close();
420         } catch (DatabaseException e) {
421             e.printStackTrace();
422         }
423     }
424
425     public synchronized V remove(final Object key) {
426         V prevValue = get(key);
427         memMap.remove(key);
428         expungeStaleEntries();
429         diskMap.remove(key);
430         diskMapSize--;
431         return prevValue;
432     }
433
434     public synchronized boolean containsKey(Object key) {
435         if (quickContainsKey(key)) {
436             return true;
437         }
438         return diskMap.containsKey(key);
439     }
440
441     public synchronized boolean quickContainsKey(Object key) {
442         expungeStaleEntries();
443         return memMap.containsKey(key);
444     }
445
446     public synchronized boolean containsValue(Object value) {
447         if (quickContainsValue(value)) {
448             return true;
449         }
450         return diskMap.containsValue(value);
451     }
452
453     public synchronized boolean quickContainsValue(Object value) {
454         expungeStaleEntries();
455         // FIXME this isn't really right, as memMap is of SoftEntries
456 return memMap.containsValue(value);
457     }
458
459     public int size() {
460         return diskMapSize;
461     }
462     
463     protected String getDatabaseName() {
464         String name = "DbName-Lookup-Failed";
465         try {
466             if (this.db != null) {
467                 name = this.db.getDatabaseName();
468             }
469         } catch (DatabaseException e) {
470             // Ignore.
471 }
472         return name;
473     }
474     
475     /**
476      * Sync in-memory map entries to backing disk store.
477      * When done, the memory map will be cleared and all entries stored
478      * on disk.
479      */
480     public synchronized void sync() {
481         String dbName = null;
482         // Sync. memory and disk.
483 long startTime = 0;
484         if (logger.isLoggable(Level.INFO)) {
485             dbName = getDatabaseName();
486             startTime = System.currentTimeMillis();
487             logger.info(dbName + " start sizes: disk " + this.diskMapSize +
488                 ", mem " + this.memMap.size());
489         }
490         expungeStaleEntries();
491         LinkedList <SoftEntry> stale = new LinkedList <SoftEntry>();
492         for (Iterator i = this.memMap.keySet().iterator(); i.hasNext();) {
493             Object key = i.next();
494             SoftEntry entry = (SoftEntry) memMap.get(key);
495             if (entry != null) {
496                 // Get & hold so not cleared pre-return.
497 Object value = entry.get();
498                 if (value != null) {
499                     this.diskMap.put(key, value);
500                 } else {
501                     stale.add(entry);
502                 }
503             }
504         }
505         // for any entries above that had been cleared, ensure expunged
506 for (SoftEntry entry : stale) {
507             expungeStaleEntry(entry);
508         }
509         
510         if (logger.isLoggable(Level.INFO)) {
511             logger.info(dbName + " sync took " +
512                 (System.currentTimeMillis() - startTime) + "ms. " +
513                 "Finish sizes: disk " +
514                 this.diskMapSize + ", mem " + this.memMap.size());
515         }
516     }
517
518     private void expungeStaleEntries() {
519         int c = 0;
520         for(SoftEntry entry; (entry = refQueuePoll()) != null;) {
521             expungeStaleEntry(entry);
522             c++;
523         }
524         if (c > 0 && logger.isLoggable(Level.FINER)) {
525             try {
526                 logger.finer("DB: " + db.getDatabaseName() + ", Expunged: "
527                         + c + ", Diskmap size: " + diskMapSize
528                         + ", Cache size: " + memMap.size());
529             } catch (DatabaseException e) {
530                 // Just for logging so ignore Exceptions
531 }
532         }
533     }
534     
535     private void expungeStaleEntry(SoftEntry entry) {
536         // If phantom already null, its already expunged -- probably
537 // because it was purged directly first from inside in
538 // {@link #get(String)} and then it went on the poll queue and
539 // when it came off inside in expungeStaleEntries, this method
540 // was called again.
541 if (entry.getPhantom() == null) {
542             return;
543         }
544         // If the object that is in memMap is not the one passed here, then
545 // memMap has been changed -- probably by a put on top of this entry.
546 if (memMap.get(entry.getPhantom().getKey()) == entry) {
547             memMap.remove(entry.getPhantom().getKey());
548             diskMap.put(entry.getPhantom().getKey(),
549                 entry.getPhantom().doctoredGet());
550         }
551         entry.clearPhantom();
552     }
553     
554     private class PhantomEntry<T> extends PhantomReference <T> {
555         private final Object key;
556
557         public PhantomEntry(Object key, T referent) {
558             super(referent, null);
559             this.key = key;
560         }
561
562         /**
563          * @return Return the referent. The contract for {@link #get()}
564          * always returns a null referent. We've cheated and doctored
565          * PhantomReference to return the actual referent value. See notes
566          * at {@link #referentField};
567          */
568         public Object doctoredGet() {
569             try {
570                 // Here we use the referentField saved off on static
571 // initialization of this class to get at this References'
572 // private referent field.
573 return referentField.get(this);
574             } catch (IllegalAccessException e) {
575                 throw new RuntimeException (e);
576             }
577         }
578
579         /**
580          * @return Returns the key.
581          */
582         public Object getKey() {
583             return this.key;
584         }
585     }
586
587     private class SoftEntry<T> extends SoftReference <T> {
588         private PhantomEntry<T> phantom;
589
590         public SoftEntry(Object key, T referent, ReferenceQueue <T> q) {
591             super(referent, q);
592             this.phantom = new PhantomEntry<T>(key, referent);
593         }
594
595         /**
596          * @return Returns the phantom reference.
597          */
598         public PhantomEntry getPhantom() {
599             return this.phantom;
600         }
601         
602         public void clearPhantom() {
603             this.phantom.clear();
604             this.phantom = null;
605             super.clear();
606         }
607     }
608     
609     private void readObject(java.io.ObjectInputStream stream)
610     throws IOException , ClassNotFoundException {
611         stream.defaultReadObject();
612         initializeInstance();
613         if (logger.isLoggable(Level.FINE)) {
614             logger.fine(getDatabaseName() + " diskMapSize: " + diskMapSize);
615         }
616     }
617     
618  
619     
620     @SuppressWarnings ("unchecked")
621     private K toKey(Object o) {
622         return (K)o;
623     }
624     
625     @SuppressWarnings ("unchecked")
626     private V diskMapGet(K k) {
627         return (V)diskMap.get(k);
628     }
629     
630     @SuppressWarnings ("unchecked")
631     private SoftEntry<V> refQueuePoll() {
632         return (SoftEntry)refQueue.poll();
633     }
634 }
635

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags