StoreJanitorImpl


1   /*
2    * Copyright 1999-2004 The Apache Software Foundation.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *      http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package org.apache.cocoon.components.store.impl;
17  
18  import java.util.ArrayList  ;
19  import java.util.Iterator  ;
20  
21  import org.apache.avalon.framework.activity.Startable;
22  import org.apache.avalon.framework.logger.AbstractLogEnabled;
23  import org.apache.avalon.framework.parameters.ParameterException;
24  import org.apache.avalon.framework.parameters.Parameterizable;
25  import org.apache.avalon.framework.parameters.Parameters;
26  import org.apache.avalon.framework.thread.ThreadSafe;
27  import org.apache.excalibur.store.Store;
28  import org.apache.excalibur.store.StoreJanitor;
29  
30  /**
31   * This class is a implentation of a StoreJanitor. Store classes
32   * can register to the StoreJanitor. When memory is too low,
33   * the StoreJanitor frees the registered caches until memory is normal.
34   *
35   * <p>A few parameters can be used:
36   * <UL>
37   *  <LI><B>freememory</B>: How many bytes shall be always free in the JVM (Default: 1mb)</LI>
38   *  <LI><B>heapsize</B>: Maximum possible size of the JVM memory consumption (Default: 64mb)</LI>
39   *  <LI><B>cleanupthreadinterval</B>: How often (sec) shall run the cleanup thread (Default: 10s)</LI>
40   *  <LI><B>adaptivethreadinterval</B> (experimental): Enable adaptive algorithm to determine thread interval
41   *      (Default: false) When true, <code>cleanupthreadinterval</code> defines the maximum cleanup interval.
42   *      Cleanup interval then is determined based on the memory fill rate: the faster memory is filled in,
43   *      and the less free memory is left, the shorter is the cleanup time.</LI>
44   *  <LI><B>threadpriority</B>: priority of the thread (1-10). (Default: 10)</LI>
45   *  <LI><B>percent_to_free</B>: What fraction of the store to free when memory is low (1-100). (Default: 10%)</LI>
46   *  <LI><B>invokegc</B>: Invoke the gc on low memory first (true|false; default: false)</LI>
47   * </UL></p>
48   *
49   * @avalon.component
50   * @avalon.service type=StoreJanitor
51   * @x-avalon.info name=store-janitor
52   * @x-avalon.lifestyle type=singleton
53   *
54   * @author <a HREF="mailto:dev@avalon.apache.org">Avalon Development Team</a>
55   * @version CVS $Id: StoreJanitorImpl.java 332021 2005-11-09 11:22:09Z crossley $
56   */
57  public class StoreJanitorImpl extends AbstractLogEnabled
58                                implements StoreJanitor, Parameterizable, ThreadSafe,
59                                           Runnable  , Startable {
60  
61      // Configuration parameters
62      private int minFreeMemory = -1;
63      private int maxHeapSize = -1;
64      private int threadInterval = -1;
65      private int minThreadInterval = 500;
66      private boolean adaptiveThreadInterval;
67      private int priority = -1;
68      private double fraction;
69  
70      private Runtime   jvm;
71      private ArrayList   storelist;
72      private int index = -1;
73  
74      /** Should the gc be called on low memory? */
75      protected boolean invokeGC;
76  
77      private boolean doRun;
78  
79      /**
80       * Amount of memory in use before sleep(). Must be initially set a resonable
81       * value; ie. <code>memoryInUse()</code>
82       */
83      protected long inUse;
84  
85      private boolean firstRun = true;
86  
87      /** The calculated delay for the next checker run in ms */
88      protected long interval = Long.MAX_VALUE;
89  
90      /** Used memory change rate in bytes per second */
91      private long maxRateOfChange = 1;
92  
93  
94      /**
95       * Parameterize the StoreJanitorImpl.
96       */
97      public void parameterize(Parameters params) throws ParameterException {
98          this.jvm = Runtime.getRuntime();
99          this.minFreeMemory = params.getParameterAsInteger("freememory", 1024 * 1024);
100         this.maxHeapSize = params.getParameterAsInteger("heapsize", 66600000);
101         // Parameter value is in seconds, converted to millis
102         this.threadInterval = params.getParameterAsInteger("cleanupthreadinterval", 10) * 1000;
103         this.adaptiveThreadInterval = params.getParameterAsBoolean("adaptivethreadinterval", false);
104         this.priority = params.getParameterAsInteger("threadpriority", Thread.currentThread().getPriority());
105         int percent = params.getParameterAsInteger("percent_to_free", 10);
106         this.invokeGC = params.getParameterAsBoolean("invokegc", this.invokeGC);
107 
108         if (getMinFreeMemory() < 1) {
109             throw new ParameterException("StoreJanitorImpl freememory parameter has to be greater then 1");
110         }
111         if (getMaxHeapSize() < 1) {
112             throw new ParameterException("StoreJanitorImpl heapsize parameter has to be greater then 1");
113         }
114         if (getThreadInterval() < 1) {
115             throw new ParameterException("StoreJanitorImpl cleanupthreadinterval parameter has to be greater then 1");
116         }
117         if (getPriority() < 1 || getPriority() > 10) {
118             throw new ParameterException("StoreJanitorImpl threadpriority has to be between 1 and 10");
119         }
120         if (percent > 100 && percent < 1) {
121             throw new ParameterException("StoreJanitorImpl percent_to_free, has to be between 1 and 100");
122         }
123 
124         this.fraction = percent / 100.0D;
125         this.storelist = new ArrayList  ();
126 
127         if (getLogger().isDebugEnabled()) {
128             getLogger().debug("minimum free memory=" + getMinFreeMemory());
129             getLogger().debug("heapsize=" + getMaxHeapSize());
130             getLogger().debug("thread interval=" + getThreadInterval());
131             getLogger().debug("adaptivethreadinterval=" + getAdaptiveThreadInterval());
132             getLogger().debug("priority=" + getPriority());
133             getLogger().debug("percent=" + percent);
134             getLogger().debug("invoke gc=" + this.invokeGC);
135         }
136     }
137 
138     public void start() throws Exception   {
139         this.doRun = true;
140         Thread   checker = new Thread  (this);
141         if (getLogger().isDebugEnabled()) {
142             getLogger().debug("Intializing checker thread");
143         }
144         checker.setPriority(getPriority());
145         checker.setDaemon(true);
146         checker.setName("checker");
147         checker.start();
148     }
149 
150     public void stop() {
151         this.doRun = false;
152     }
153 
154     /**
155      * The "checker" thread loop.
156      */
157     public void run() {
158         this.inUse = memoryInUse();
159         while (this.doRun) {
160             checkMemory();
161 
162             // Sleep
163             if (getLogger().isDebugEnabled()) {
164                 getLogger().debug("Sleeping for " + this.interval + "ms");
165             }
166             try {
167                 Thread.sleep(this.interval);
168             } catch (InterruptedException   ignore) {
169             }
170 
171             // Ignore change in memory during the first run (startup)
172             if (this.firstRun) {
173                 this.firstRun = false;
174                 this.inUse = memoryInUse();
175             }
176         }
177     }
178 
179     /**
180      * The "checker" thread checks if memory is running low in the jvm.
181      */
182     protected void checkMemory() {
183         if (getAdaptiveThreadInterval()) {
184             // Monitor the rate of change of heap in use.
185             long change = memoryInUse() - inUse;
186             long rateOfChange = longDiv(change * 1000, interval); // bps.
187             if (maxRateOfChange < rateOfChange) {
188                 maxRateOfChange = (maxRateOfChange + rateOfChange) / 2;
189             }
190             if (getLogger().isDebugEnabled()) {
191                 getLogger().debug("Waking after " + interval + "ms, in use change "
192                                   + change + "b to " + memoryInUse() + "b, rate "
193                                   + rateOfChange + "b/sec, max rate " + maxRateOfChange + "b/sec");
194             }
195         }
196 
197         // Amount of memory used is greater than heapsize
198         if (memoryLow()) {
199             if (this.invokeGC) {
200                 freePhysicalMemory();
201             }
202 
203             synchronized (this) {
204                 if (!this.invokeGC
205                         || (memoryLow() && getStoreList().size() > 0)) {
206 
207                     freeMemory();
208                     setIndex(getIndex() + 1);
209                 }
210             }
211         }
212 
213         if (getAdaptiveThreadInterval()) {
214             // Calculate sleep interval based on the change rate and free memory left
215             interval = minTimeToFill(maxRateOfChange) * 1000 / 2;
216             if (interval > this.threadInterval) {
217                 interval = this.threadInterval;
218             } else if (interval < this.minThreadInterval) {
219                 interval = this.minThreadInterval;
220             }
221             inUse = memoryInUse();
222         } else {
223             interval = this.threadInterval;
224         }
225     }
226 
227     /**
228      * Method to check if memory is running low in the JVM.
229      *
230      * @return true if memory is low
231      */
232     private boolean memoryLow() {
233         if (getLogger().isDebugEnabled()) {
234             getLogger().debug("JVM Memory total: " + getJVM().totalMemory()
235                               + ", free: " + getJVM().freeMemory());
236         }
237 
238         if ((getJVM().totalMemory() >= getMaxHeapSize())
239                 && (getJVM().freeMemory() < getMinFreeMemory())) {
240             if (getLogger().isDebugEnabled()) {
241                 getLogger().debug("Memory is low!");
242             }
243             return true;
244         } else {
245             return false;
246         }
247     }
248 
249     /**
250      * Calculate the JVM memory in use now.
251      *
252      * @return memory in use.
253      */
254     protected long memoryInUse() {
255         return jvm.totalMemory() - jvm.freeMemory();
256     }
257 
258     /**
259      * Calculate amount of time needed to fill all free memory with given
260      * fill rate.
261      *
262      * @param rate memory fill rate in time per bytes
263      * @return amount of time to fill all the memory with given fill rate
264      */
265     private long minTimeToFill(long rate) {
266         return longDiv(jvm.freeMemory(), rate);
267     }
268 
269     private long longDiv(long top, long bottom) {
270         try {
271             return top / bottom;
272         } catch (Exception   e) {
273             return top > 0 ? Long.MAX_VALUE : Long.MIN_VALUE;
274         }
275     }
276 
277     /**
278      * This method register the stores
279      *
280      * @param store the store to be registered
281      */
282     public synchronized void register(Store store) {
283         getStoreList().add(store);
284         if (getLogger().isDebugEnabled()) {
285             getLogger().debug("Registered store instance " + store + ". Stores now: "
286                               + getStoreList().size());
287         }
288     }
289 
290     /**
291      * This method unregister the stores
292      *
293      * @param store the store to be unregistered
294      */
295     public synchronized void unregister(Store store) {
296         getStoreList().remove(store);
297         if (getLogger().isDebugEnabled()) {
298             getLogger().debug("Unregistered store instance " + store + ". Stores now: "
299                               + getStoreList().size());
300         }
301     }
302 
303     /**
304      * This method return a java.util.Iterator of every registered stores
305      *
306      * <i>The iterators returned is fail-fast: if list is structurally
307      * modified at any time after the iterator is created, in any way, the
308      * iterator will throw a ConcurrentModificationException.  Thus, in the
309      * face of concurrent modification, the iterator fails quickly and
310      * cleanly, rather than risking arbitrary, non-deterministic behavior at
311      * an undetermined time in the future.</i>
312      *
313      * @return a java.util.Iterator
314      */
315     public Iterator   iterator() {
316         return getStoreList().iterator();
317     }
318 
319     /**
320      * Round Robin alghorithm for freeing the registered caches.
321      */
322     private void freeMemory() {
323         // TODO: Alternative to RR might be to free same fraction from every storage.
324         try {
325             // Determine the store.
326             if (getIndex() < getStoreList().size()) {
327                 if (getIndex() == -1) {
328                     setIndex(0);
329                 }
330             } else {
331                 // Store list changed (one or more store has been removed).
332                 if (getLogger().isDebugEnabled()) {
333                     getLogger().debug("Restarting from the beginning");
334                 }
335                 setIndex(0);
336             }
337 
338             // Delete proportionate elements out of the store as configured.
339             Store store = (Store)getStoreList().get(getIndex());
340             int limit = calcToFree(store);
341             if (getLogger().isDebugEnabled()) {
342                 getLogger().debug("Freeing " + limit + " items from store #" + getIndex());
343             }
344 
345             for (int i = 0; i < limit; i++) {
346                 try {
347                     store.free();
348                 } catch (OutOfMemoryError   e) {
349                     getLogger().error("OutOfMemoryError in freeMemory()");
350                 }
351             }
352         } catch (Exception   e) {
353             getLogger().error("Error in freeMemory()", e);
354         } catch (OutOfMemoryError   e) {
355             getLogger().error("OutOfMemoryError in freeMemory()");
356         }
357     }
358 
359     /**
360      * This method claculates the number of Elements to be freememory
361      * out of the Cache.
362      *
363      * @param store the Store which was selected as victim
364      * @return number of elements to be removed!
365      */
366     private int calcToFree(Store store) {
367         int cnt = store.size();
368         if (cnt < 0) {
369             if (getLogger().isDebugEnabled()) {
370                 getLogger().debug("Unknown size of the store: " + store);
371             }
372             return 0;
373         }
374 
375         final int res = (int) (cnt * fraction);
376         if (getLogger().isDebugEnabled()) {
377             getLogger().debug("Calculating size for store " + store + " with size " + cnt + ": " + res);
378         }
379         return res;
380     }
381 
382     /**
383      * This method forces the garbage collector
384      */
385     private void freePhysicalMemory() {
386         if (getLogger().isDebugEnabled()) {
387             getLogger().debug("Invoking GC. Memory total: "
388                               + getJVM().totalMemory() + ", free: "
389                               + getJVM().freeMemory());
390         }
391 
392         getJVM().runFinalization();
393         getJVM().gc();
394 
395         if (getLogger().isDebugEnabled()) {
396             getLogger().debug("GC complete. Memory total: "
397                               + getJVM().totalMemory() + ", free: "
398                               + getJVM().freeMemory());
399         }
400     }
401 
402 
403     private int getMinFreeMemory() {
404         return this.minFreeMemory;
405     }
406 
407     private int getMaxHeapSize() {
408         return this.maxHeapSize;
409     }
410 
411     private int getPriority() {
412         return this.priority;
413     }
414 
415     private int getThreadInterval() {
416         return this.threadInterval;
417     }
418 
419     private boolean getAdaptiveThreadInterval() {
420         return this.adaptiveThreadInterval;
421     }
422 
423     private Runtime   getJVM() {
424         return this.jvm;
425     }
426 
427     private ArrayList   getStoreList() {
428         return this.storelist;
429     }
430 
431     private void setIndex(int _index) {
432         if (getLogger().isDebugEnabled()) {
433             getLogger().debug("Setting index=" + _index);
434         }
435         this.index = _index;
436     }
437 
438     private int getIndex() {
439         return this.index;
440     }
441 }
442
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags