StoreJanitorImpl


1   /* 
2    * Copyright 2002-2004 The Apache Software Foundation
3    * Licensed  under the  Apache License,  Version 2.0  (the "License");
4    * you may not use  this file  except in  compliance with the License.
5    * You may obtain a copy of the License at 
6    * 
7    *   http://www.apache.org/licenses/LICENSE-2.0
8    * 
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed  under the  License is distributed on an "AS IS" BASIS,
11   * WITHOUT  WARRANTIES OR CONDITIONS  OF ANY KIND, either  express  or
12   * implied.
13   * 
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.excalibur.store.impl;
18  
19  import java.util.ArrayList  ;
20  import java.util.Iterator  ;
21  
22  import org.apache.avalon.framework.activity.Startable;
23  import org.apache.avalon.framework.logger.AbstractLogEnabled;
24  import org.apache.avalon.framework.parameters.ParameterException;
25  import org.apache.avalon.framework.parameters.Parameterizable;
26  import org.apache.avalon.framework.parameters.Parameters;
27  import org.apache.avalon.framework.thread.ThreadSafe;
28  import org.apache.excalibur.store.Store;
29  import org.apache.excalibur.store.StoreJanitor;
30  
31  /**
32   * This class is a implentation of a StoreJanitor. Store classes
33   * can register to the StoreJanitor. When memory is too low,
34   * the StoreJanitor frees the registered caches until memory is normal.
35   * 
36   * @avalon.component
37   * @avalon.service type=StoreJanitor
38   * @x-avalon.info name=store-janitor
39   * @x-avalon.lifestyle type=singleton
40   *
41   * @author <a HREF="mailto:dev@avalon.apache.org">Avalon Development Team</a>
42   * @version CVS $Id: StoreJanitorImpl.java,v 1.4 2004/02/28 11:47:31 cziegeler Exp $
43   */
44  public class StoreJanitorImpl
45  extends AbstractLogEnabled
46  implements StoreJanitor,
47             Parameterizable,
48             ThreadSafe,
49             Runnable  ,
50             Startable
51  {
52  
53      private boolean doRun = false;
54  
55      // Configuration parameters
56      private int minFreeMemory = -1;
57      private int maxHeapSize = -1;
58      private int threadInterval = -1;
59      private int minThreadInterval = 500;
60      private boolean adaptiveThreadInterval = false;
61      private int priority = -1;
62      private double fraction;
63  
64      private Runtime   jvm;
65      private ArrayList   storelist;
66      private int index = -1;
67      /** Should the gc be called on low memory? */
68      protected boolean invokeGC = false;
69      
70      /**
71       * Initialize the StoreJanitorImpl.
72       * A few options can be used :
73       * <UL>
74       *  <LI><B>freememory</B>: How many bytes shall be always free in the JVM (Default: 1mb)</LI>
75       *  <LI><B>heapsize</B>: Maximum possible size of the JVM memory consumption (Default: 64mb)</LI>
76       *  <LI><B>cleanupthreadinterval</B>: How often (sec) shall run the cleanup thread (Default: 10s)</LI>
77       *  <LI><B>adaptivethreadinterval</B> (experimental): Enable adaptive algorithm to determine thread interval
78       *      (Default: false) When true, <code>cleanupthreadinterval</code> defines the maximum cleanup interval.
79       *      Cleanup interval then is determined based on the memory fill rate: the faster memory is filled in,
80       *      and the less free memory is left, the shorter is the cleanup time.</LI>
81       *  <LI><B>threadpriority</B>: priority of the thread (1-10). (Default: 10)</LI>
82       *  <LI><B>percent_to_free</B>: What fraction of the store to free when memory is low (1-100). (Default: 10%)</LI>
83       *  <LI><B>invokegc</B>: Invoke the gc on low memory first (true|false; default: false)</LI>
84       * </UL>
85       *
86       * @param params the Configuration of the application
87       * @exception ParameterException
88       */
89      public void parameterize(Parameters params) throws ParameterException 
90      {
91          if (getLogger().isDebugEnabled()) 
92          {
93              getLogger().debug("Configure StoreJanitorImpl");
94          }
95          setJVM(Runtime.getRuntime());
96  
97          setMinFreeMemory(params.getParameterAsInteger("freememory", 1024 * 1024));
98          setMaxHeapSize(params.getParameterAsInteger("heapsize", 60 * 1024 * 1024));
99          // Parameter value is in seconds, converted to millis
100         setThreadInterval(params.getParameterAsInteger("cleanupthreadinterval", 10) * 1000);
101         setAdaptiveThreadInterval(params.getParameterAsBoolean("adaptivethreadinterval", false));
102         setPriority(params.getParameterAsInteger("threadpriority",
103                                                  Thread.currentThread().getPriority()));
104         int percent = params.getParameterAsInteger("percent_to_free", 10);
105         this.invokeGC = params.getParameterAsBoolean("invokegc", this.invokeGC);
106         
107         if (getMinFreeMemory() < 1) 
108         {
109             throw new ParameterException("StoreJanitorImpl freememory parameter has to be greater then 1");
110         }
111         if (getMaxHeapSize() < 1) 
112         {
113             throw new ParameterException("StoreJanitorImpl heapsize parameter has to be greater then 1");
114         }
115         if (getThreadInterval() < 1) 
116         {
117             throw new ParameterException("StoreJanitorImpl cleanupthreadinterval parameter has to be greater then 1");
118         }
119         if (getPriority() < 1 || getPriority() > 10) 
120         {
121             throw new ParameterException("StoreJanitorImpl threadpriority has to be between 1 and 10");
122         }
123         if (percent > 100 && percent < 1) 
124         {
125             throw new ParameterException("StoreJanitorImpl percent_to_free, has to be between 1 and 100");
126         }
127 
128         this.fraction = percent / 100.0D;
129         setStoreList(new ArrayList  ());
130         
131         if ( getLogger().isDebugEnabled() ) 
132         {
133             getLogger().debug("minimum free memory=" + this.getMinFreeMemory());
134             getLogger().debug("heapsize=" + this.getMaxHeapSize());
135             getLogger().debug("thread interval=" + this.getThreadInterval());
136             getLogger().debug("priority=" + this.getPriority());
137             getLogger().debug("percent=" + percent);
138             getLogger().debug("invoke gc=" + this.invokeGC);
139         }
140     }
141 
142     public void start() 
143     {
144         doRun = true;
145         Thread   checker = new Thread  (this);
146         if (getLogger().isDebugEnabled()) 
147         {
148             getLogger().debug("Intializing checker thread");
149         }
150         checker.setPriority(getPriority());
151         checker.setDaemon(true);
152         checker.setName("checker");
153         checker.start();
154     }
155 
156     public void stop() 
157     {
158         doRun = false;
159     }
160 
161     /**
162      * The "checker" thread checks if memory is running low in the jvm.
163      */
164     public void run() 
165     {
166         boolean firstRun = true;
167         long inUse = memoryInUse(); // Amount of memory in use before sleep()
168         long interval = Long.MAX_VALUE; // Sleep time in ms
169         long maxRateOfChange = 1; // Used memory change rate in bytes per second
170 
171         while (doRun) {
172             if (getAdaptiveThreadInterval()) 
173             {
174                 // Monitor the rate of change of heap in use.
175                 long change = memoryInUse() - inUse;
176                 long rateOfChange = longDiv(change * 1000, interval); // bps.
177                 if (maxRateOfChange < rateOfChange) 
178                 {
179                     maxRateOfChange = (maxRateOfChange + rateOfChange) / 2;
180                 }
181                 if (getLogger().isDebugEnabled()) {
182                     getLogger().debug("Waking after " + interval + "ms, in use change "
183                                       + change + "b to " + memoryInUse() + "b, rate "
184                                       + rateOfChange + "b/sec, max rate " + maxRateOfChange + "b/sec");
185                 }
186             }
187 
188             // Amount of memory used is greater than heapsize
189             if (memoryLow()) 
190             {
191                 if ( this.invokeGC ) 
192                 {
193                     this.freePhysicalMemory();
194                 }
195 
196                 synchronized (this) 
197                 {
198                     if (!this.invokeGC
199                         || (memoryLow() && getStoreList().size() > 0)) 
200                     {
201                             
202                         freeMemory();
203                         setIndex(getIndex() + 1);
204                     }
205                 }
206             }
207 
208             if (getAdaptiveThreadInterval()) 
209             {
210                 // Calculate sleep interval based on the change rate and free memory left
211                 interval = minTimeToFill(maxRateOfChange) * 1000 / 2;
212                 if (interval > this.threadInterval) 
213                 {
214                     interval = this.threadInterval;
215                 } 
216                 else if (interval < this.minThreadInterval) 
217                 {
218                     interval = this.minThreadInterval;
219                 }
220                 inUse = memoryInUse();
221             } 
222             else 
223             {
224                 interval = this.threadInterval;
225             }
226             if (getLogger().isDebugEnabled()) 
227             {
228                 getLogger().debug("Sleeping for " + interval + "ms");
229             }
230 
231             // Sleep
232             try 
233             {
234                 Thread.sleep(interval);
235             } 
236             catch (InterruptedException   ignore) {}
237 
238             // Ignore change in memory during the first run (startup)
239             if (firstRun) 
240             {
241                 firstRun = false;
242                 inUse = memoryInUse();
243             }
244         }
245     }
246 
247     /**
248      * Method to check if memory is running low in the JVM.
249      *
250      * @return true if memory is low
251      */
252     private boolean memoryLow() 
253     {
254         if (getLogger().isDebugEnabled()) 
255         {
256             getLogger().debug("JVM Memory total: " + getJVM().totalMemory()
257                               + ", free: " + getJVM().freeMemory());
258         }
259 
260         if ((getJVM().totalMemory() >= getMaxHeapSize())
261                 && (getJVM().freeMemory() < getMinFreeMemory())) 
262         {
263             if (getLogger().isDebugEnabled()) 
264             {
265                 getLogger().debug("Memory is low!");
266             }
267             return true;
268         } 
269         else 
270         {
271             return false;
272         }
273     }
274 
275     /**
276      * Calculate the JVM memory in use now.
277      *
278      * @return memory in use.
279      */
280     private long memoryInUse() 
281     {
282         return jvm.totalMemory() - jvm.freeMemory();
283     }
284 
285     /**
286      * Calculate amount of time needed to fill all free memory with given
287      * fill rate.
288      *
289      * @param rate memory fill rate in time per bytes
290      * @return amount of time to fill all the memory with given fill rate
291      */
292     private long minTimeToFill(long rate) 
293     {
294         return longDiv(jvm.freeMemory(), rate);
295     }
296 
297     private long longDiv(long top, long bottom) 
298     {
299         try 
300         {
301             return top / bottom;
302         } 
303         catch (Exception   e) 
304         {
305             return top > 0 ? Long.MAX_VALUE : Long.MIN_VALUE;
306         }
307     }
308 
309     /**
310      * This method register the stores
311      *
312      * @param store the store to be registered
313      */
314     public synchronized void register(Store store) 
315     {
316         getStoreList().add(store);
317         if (getLogger().isDebugEnabled()) 
318         {
319             getLogger().debug("Registered store instance " + store + ". Stores now: "
320                               + getStoreList().size());
321         }
322     }
323 
324     /**
325      * This method unregister the stores
326      *
327      * @param store the store to be unregistered
328      */
329     public synchronized void unregister(Store store) 
330     {
331         getStoreList().remove(store);
332         if (getLogger().isDebugEnabled()) 
333         {
334             getLogger().debug("Unregistered store instance " + store + ". Stores now: "
335                               + getStoreList().size());
336         }
337     }
338 
339     /**
340      * This method return a java.util.Iterator of every registered stores
341      *
342      * <i>The iterators returned is fail-fast: if list is structurally
343      * modified at any time after the iterator is created, in any way, the
344      * iterator will throw a ConcurrentModificationException.  Thus, in the
345      * face of concurrent modification, the iterator fails quickly and
346      * cleanly, rather than risking arbitrary, non-deterministic behavior at
347      * an undetermined time in the future.</i>
348      *
349      * @return a java.util.Iterator
350      */
351     public Iterator   iterator() 
352     {
353         return getStoreList().iterator();
354     }
355 
356     /**
357      * Round Robin alghorithm for freeing the registered caches.
358      */
359     private void freeMemory() 
360     {
361         // TODO: Alternative to RR might be to free same fraction from every storage.
362         try 
363         {
364             // Determine the store.
365             if (getIndex() < getStoreList().size()) 
366             {
367                 if (getIndex() == -1) 
368                 {
369                     setIndex(0);
370                 }
371             } 
372             else 
373             {
374                 // Store list changed (one or more store has been removed).
375                 if (getLogger().isDebugEnabled()) 
376                 {
377                     getLogger().debug("Restarting from the beginning");
378                 }
379                 setIndex(0);
380             }
381 
382             // Delete proportionate elements out of the store as configured.
383             Store store = (Store)getStoreList().get(getIndex());
384             int limit = calcToFree(store);
385             if (getLogger().isDebugEnabled()) 
386             {
387                 getLogger().debug("Freeing " + limit + " items from store N " + getIndex());
388             }
389             for (int i=0; i < limit; i++) 
390             {
391                 try
392                 {
393                     store.free();
394                 } 
395                 catch (OutOfMemoryError   e) 
396                 {
397                     getLogger().error("OutOfMemoryError in freeMemory()");
398                 }
399             }
400         } 
401         catch (Exception   e) 
402         {
403             getLogger().error("Error in freeMemory()", e);
404         } 
405         catch (OutOfMemoryError   e) 
406         {
407             getLogger().error("OutOfMemoryError in freeMemory()");
408         }
409     }
410 
411     /**
412      * This method claculates the number of Elements to be freememory
413      * out of the Cache.
414      *
415      * @param store the Store which was selected as victim
416      * @return number of elements to be removed!
417      */
418     private int calcToFree(Store store) 
419     {
420         int cnt = store.size();
421         if (cnt < 0) 
422         {
423             if ( getLogger().isDebugEnabled() )
424             {
425                 getLogger().debug("Unknown size of the store: " + store);
426             }
427             return 0;
428         }
429         final int res = (int)(cnt * fraction);
430         if ( getLogger().isDebugEnabled() ) 
431         {
432             getLogger().debug("Calculating size for store " + store + " with size " + cnt + " : " + res);
433         }
434         return res;
435     }
436 
437     /**
438      * This method forces the garbage collector
439      */
440     private void freePhysicalMemory() 
441     {
442         if (getLogger().isDebugEnabled()) 
443         {
444             getLogger().debug("Invoking garbage collection. Memory total: "
445                               + getJVM().totalMemory() + ", free: "
446                               + getJVM().freeMemory());
447         }
448 
449         getJVM().runFinalization();
450         getJVM().gc();
451 
452         if (getLogger().isDebugEnabled()) 
453         {
454             getLogger().debug("Garbage collection complete. Memory total: "
455                               + getJVM().totalMemory() + ", free: "
456                               + getJVM().freeMemory());
457         }
458     }
459      
460 
461     private int getMinFreeMemory() 
462     {
463         return this.minFreeMemory;
464     }
465 
466     private void setMinFreeMemory(int _freememory) 
467     {
468         this.minFreeMemory = _freememory;
469     }
470 
471     private int getMaxHeapSize() 
472     {
473         return this.maxHeapSize;
474     }
475 
476     private void setMaxHeapSize(int _heapsize) 
477     {
478         this.maxHeapSize = _heapsize;
479     }
480 
481     private int getPriority() 
482     {
483         return this.priority;
484     }
485 
486     private void setPriority(int _priority) 
487     {
488         this.priority = _priority;
489     }
490 
491     private int getThreadInterval() 
492     {
493         return this.threadInterval;
494     }
495 
496     private void setThreadInterval(int _threadInterval) 
497     {
498         this.threadInterval = _threadInterval;
499     }
500 
501     private boolean getAdaptiveThreadInterval()
502     {
503         return this.adaptiveThreadInterval;
504     }
505 
506     private void setAdaptiveThreadInterval(boolean _adaptiveThreadInterval) 
507     {
508         this.adaptiveThreadInterval = _adaptiveThreadInterval;
509     }
510 
511     private Runtime   getJVM() 
512     {
513         return this.jvm;
514     }
515 
516     private void setJVM(Runtime   _jvm) 
517     {
518         this.jvm = _jvm;
519     }
520 
521     private ArrayList   getStoreList() 
522     {
523         return this.storelist;
524     }
525 
526     private void setStoreList(ArrayList   _storelist) 
527     {
528         this.storelist = _storelist;
529     }
530 
531     private void setIndex(int _index) 
532     {
533         if (getLogger().isDebugEnabled()) 
534         {
535             getLogger().debug("Setting index=" + _index);
536         }
537         this.index = _index;
538     }
539 
540     private int getIndex() 
541     {
542         return this.index;
543     }
544 }
545
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags