Java > Open Source Codes > org > mmbase > module > scancache


1 /*
2
3 This software is OSI Certified Open Source Software.
4 OSI Certified is a certification mark of the Open Source Initiative.
5
6 The license (Mozilla version 1.0) can be read at the MMBase site.
7 See http://www.MMBase.org/license
8
9 */
10 package org.mmbase.module;
11
12 import java.io.*;
13 import java.sql.Date ;
14 import java.util.Hashtable ;
15
16 import javax.servlet.http.HttpServletResponse ;
17
18 import org.mmbase.module.builders.NetFileSrv;
19 import org.mmbase.module.core.MMBase;
20 import org.mmbase.module.gui.html.scanparser;
21 import org.mmbase.util.*;
22 import org.mmbase.util.logging.*;
23
24 /**
25  * File cache system.
26  * This system only caches texts (it stores and retrieves strings).
27  * Texts are asociated with a key, which is used both as a filename on disk and a
28  * key into a memory cache.
29  * While in theory it is possible to cache ANY text, this module is mainly used to store pages
30  * based on their url.<br />
31  * Caching is done in pools. Each pool has its own memory cache and files, and has
32  * different ways to handle file caching. The pools currently supported are "PAGE" and "HENK".
33  *
34  * @application SCAN
35  * @rename SCANCache
36  * @author Daniel Ockeloen
37  * @author Pierre van Rooden (javadocs)
38  * @version $Id: scancache.java,v 1.42 2005/09/20 19:28:29 nklasens Exp $
39  */
40 public class scancache extends Module implements scancacheInterface {
41
42     /**
43      * Maximum size of a pool's memory cache.
44      */
45     public static final int MAX_CACHE_POOL_SIZE = 200;
46
47     // logger
48 private static Logger log = Logging.getLoggerInstance(scancache.class.getName());
49
50         private scanparser scanparser;
51
52     /**
53      * Default expiration time for a cache entry, in seconds.
54      * The default is 6 hours.
55      * Should be made configurable through a property in scancache.xml
56      */
57     private static int defaultExpireTime = 21600;
58
59     /**
60      * Contains the memory caches for the cache pools.
61      * Pools are identified by name. Various pools have different ways of cache-handling.
62      * The pools currently supported are 'HENK' and 'PAGE'.
63      * The key to retrieve a pool is the poolname. The value returned ia a LRUHashtable,
64      * configured to hold a maximum of MAX_CACHE_POOL_SIZE entries.
65      */
66     Hashtable pools = new Hashtable ();
67
68     /**
69      * Contains the last date (as an <code>Integer</code>) a file was stored in a pool.
70      * The key to retrieve the time is poolname+filekey (where a filekey is generally the file URI).
71      * There is a limit to the number of values stored in this pool. This means if you have
72          * more then 4 pooltypes you have to bump that value or suffer performance degredation
73      */
74     LRUHashtable timepool = new LRUHashtable(MAX_CACHE_POOL_SIZE*4);
75
76     // org.mmbas StatisticsInterface stats;
77
78     /**
79      * reference to MMBase module, used to retrieve netfiles and pagemakers builders
80      * that support caching.
81      */
82     MMBase mmbase;
83     /**
84      * Determines whether the cache module is active.
85      * Set by the status field in the scancache.xml configuration file.
86      */
87     boolean status=false;
88
89     /**
90      * The root of the cache filepath.
91      * Set by the CacheRoot property in the scancache.xml configuration file.
92      */
93     private String cachepath="";
94
95     /**
96      * Scancache module constructor.
97      */
98     public scancache() {
99     }
100
101     /**
102      * Event that should be triggered when the module is loaded.
103      * Unused.
104      */
105     public void onload() {
106     }
107
108     /**
109      * Event that sh*ould be triggered when the module is shut down.
110      * Unused.
111      */
112     public void shutdown() {
113     }
114
115     /**
116      * Event that should be triggered when the module is unloaded.
117      * Unused.
118      */
119     public void unload() {
120     }
121
122     /**
123      * Initializes the module.
124      * Reads parameters from the scancache.xml configuration file.
125      */
126     public void init() {
127         String tmp=getInitParameter("status");
128         log.debug("status "+tmp);
129         if (tmp!=null && tmp.equals("active")) status=true;
130
131         cachepath=getInitParameter("CacheRoot");
132         if (cachepath==null) {
133             // XXX should set cache to inactive?
134 log.error("SCANCACHE -> No CacheRoot property set in the scancache.xml configuration file");
135         }
136         mmbase=(MMBase)getModule("MMBASEROOT");
137         /* org.mmbase
138         if (statmode!=null && statmode.equals("yes")) {
139             stats=(StatisticsInterface)getModule("STATS");
140         } else {
141             stats=null;
142         }
143         */
144         scanparser=(scanparser)getModule("SCANPARSER");
145     }
146
147     /**
148      * Retrieve a file from the indicated pool's cache.
149      * When using the "HENK" pool, this method performs a check on expiration using
150      * the default expiration time (6 hours).
151      * For cache "PAGE", this method does not check expiratio, and retrieves the data
152      * from the file from disk, NOT from the memory cache.
153      * @param poolname name of the cache pool, either "HENK" or "PAGE"
154      * @param key URL of the page to retrieve
155      * @return the page's content as a string, or <code>null</code> if no entry was found
156      * (i.e. cache was empty or poolname was invalid).
157      */
158     public String get(String poolName, String key,scanpage sp) {
159         if (status==false) return null; // no cache when inactive
160 log.debug("poolName="+poolName+" key="+key);
161         if (poolName.equals("HENK")) {
162             String tmp=get(poolName,key,">",sp);
163             return tmp;
164         } else if (poolName.equals("PAGE")) {
165             return getPage(poolName,key);
166         }
167         log.error("get("+poolName+","+key+"): poolname("+poolName+") is an unknown cachetype!");
168         return null;
169     }
170
171     /**
172      * Retrieve a file from the indicated pool's cache.
173      * When using the "HENK" pool, this method performs a check on expiration using
174      * the default expiration time (6 hours).
175      * For cache "PAGE", this method does not check expiration, and retrieves the data
176      * from the file from disk, NOT from the memory cache.
177      * @param poolname name of the cache pool, either "HENK" or "PAGE"
178      * @param key URL of the page to retrieve
179      * @param line contains parameters for counters, expiration time, etc, in tag-format.
180      * For this method, these options are currently only used at the mmbase.org site.
181      * @return the page's content as a string, or <code>null</code> if no entry was found
182      * (i.e. cache was empty or poolname was invalid).
183      */
184 /* public String getNew(String poolName, String key,String line,scanpage sp) {
185         if (status==false) return null; // no cache when inactive
186         line=line.substring(0,line.indexOf('>'));
187
188         // org.mmbase
189 // log.debug("scancache -> new poolName="+poolName+" key="+key+" line="+line+" tagger="+counter+" stats="+stats);
190 // if (counter!=null && stats!=null) {
191 // stats.setCount(counter,1);
192 // }
193
194
195         if (poolName.equals("HENK")) {
196             String tmp=get(poolName,key,">",sp);
197             return tmp;
198         } else if (poolName.equals("PAGE")) {
199             return getPage(poolName,key);
200         }
201         log.error("getNew("+poolName+","+key+"): poolname("+poolName+") is an unknown cachetype!");
202         return null;
203     }
204 */
205     /**
206      * Retrieve a file from the indicated pool's cache.
207      * It is first retrieved from meory. if that fails, the file is retrieved from disk.
208      * This method performs a check on expiration using either the default expiration
209      * time (6 hours), or the value in the line parameter.
210          * This method returns an old version of the page if the page in the cache has expired.
211          * It will signal the scanparser to calculate a new one in the background.
212          * This avoids contention on a busy server as the page is only calculated once when expired
213          * not calculate every request that comes in during the window of calculation.
214      * @param poolname name of the cache pool, expected (but not verified) are "HENK" or "PAGE"
215      * @param key URL of the page to retrieve
216      * @param line the expiration value, either the expiration value in seconds or the word 'NOEXPIRE'. Due to
217      * legacy this value needs to be closed with a '>'.
218      * If the parameter is empty a default value is used.
219      * @return the page's content as a string, or <code>null</code> if no entry was found
220      */
221     public String get(String poolName, String key, String line,scanpage sp) {
222         if (status==false) return null; // no cache when inactive
223
224         // get the interval time
225 // (nothing, NOEXPIRE, or an int value)
226 String tmp=line.substring(0,line.indexOf('>')).trim();
227                 int interval = getExpireInterval(tmp);
228
229         int now=(int)(System.currentTimeMillis()/1000);
230
231         // get pool memory cache
232 LRUHashtable pool=(LRUHashtable)pools.get(poolName);
233         if (pool!=null) {
234             String value=(String )pool.get(key);
235             // Check expiration
236 // XXX better to check value==null first...
237 if (value!=null) {
238                     try {
239                         // get the time from memory
240 Integer tmp2=(Integer )timepool.get(poolName+key);
241                         int then=tmp2.intValue();
242                         log.debug("scancache -> file="+then+" now="+now
243                                 +" now-then="+(now-then)+" interval="+interval);
244                         if (((now-then)-interval)<0) {
245                             return value;
246                         } else {
247                                                 // Don't handle flasvar
248 if (key.indexOf(".flashvar")<0) {
249                                                         log.debug("get("+poolName+","+key+","+line+"): Request is expired, return old version");
250                                                         // RICO signal page-processor
251 signalProcessor(sp,key);
252                                                         return value;
253                                                 } else return null;
254                         }
255                     } catch(Exception e) {}
256                         }
257         }
258         // not in memorycache, get directly from file instead
259 fileInfo fileinfo=loadFile(poolName,key);
260         if (fileinfo!=null && fileinfo.value!=null) {
261             if (((now-fileinfo.time)-interval)>0) {
262                                 if (key.indexOf(".flashvar")>=0) { // Don't handle flashvar;
263 return null;
264                                 }
265                  log.debug("get("+poolName+","+key+","+line+"): Diskfile expired for file("+fileinfo.time+"), now("+now+") and interval("+interval+") , return old version ");
266                                 // RICO signal page-processor
267 signalProcessor(sp,key);
268             }
269             if (pool==null) {
270                 // create a new pool
271 pool=new LRUHashtable(MAX_CACHE_POOL_SIZE);
272                 pools.put(poolName,pool);
273             }
274             pool.put(key,fileinfo.value); // store value in the memory cache
275 timepool.put(poolName+key,new Integer (fileinfo.time)); // store expiration time
276 return fileinfo.value;
277         }
278         return null;
279     }
280
281         /**
282          * getExpireDate.
283          * @param poolName
284          * @param key
285          * @param expireStr
286          * @return long
287          */
288         public long getExpireDate(String poolName, String key, String expireStr) {
289                 int interval = getExpireInterval(expireStr);
290                 return getLastModDate(poolName, key) + (interval * 1000);
291         }
292
293         /**
294          * getLastModDate.
295          * @param poolName
296          * @param key
297          * @return long
298          */
299         public long getLastModDate(String poolName, String key) {
300                 if (timepool.containsKey(poolName+key)) {
301                         Integer tmp2=(Integer )timepool.get(poolName+key);
302                         log.debug("scancache last modified in timepool " + (tmp2.intValue()));
303                         return ((long)tmp2.intValue()) * 1000;
304                 }
305                 log.debug("scancache last modified NOT in timepool");
306                 return 0; //don't know
307 }
308
309         /**
310          * getExpireInterval.
311          * @param cacheExpire
312          * @return int
313          */
314         private int getExpireInterval(String cacheExpire) {
315                 int interval;
316                 if ("".equals(cacheExpire)) {
317                          interval = defaultExpireTime;
318                 }
319                 else {
320                         if ("NOEXPIRE".equals(cacheExpire.toUpperCase())) {
321                                 interval = Integer.MAX_VALUE;
322                         }
323                         else {
324                                 try {
325                                 interval = Integer.parseInt(cacheExpire);
326                         }
327                         catch (NumberFormatException n) {
328                                 log.error("Number format exception for expiration time ("+cacheExpire+")");
329                                 interval = defaultExpireTime;
330                         }
331                         }
332                 }
333                 log.debug("scancache expire interval: " + interval);
334                 return interval;
335         }
336
337     /**
338      * Retrieve a file from disk.
339      * This method loads a file from disk and returns the contents as a string.
340      * It does not use the memory cache of a poolname, nor does it check for
341      * expiration of the cache.
342      * Also, it does not perform updates on the memory cache.
343      * @param poolname name of the cache pool (expected, but not verified is "PAGE")
344      * @param key URL of the page to retrieve
345      * @param line cache line options, unspecified
346      * @return the page's content as a string, or <code>null</code> if no entry was found
347      */
348     private String getPage(String poolName, String key) {
349         if (status==false) return null;
350         fileInfo fileinfo=loadFile(poolName,key);
351         if (fileinfo!=null && fileinfo.value!=null) {
352             return fileinfo.value;
353         } else {
354             return null;
355         }
356     }
357
358     /**
359      * Store a file in the indicated pool's cache (both on file system and in the memory cache).
360      * Returns the old value if available.
361      * When using the "HENK" pool, this method performs a check on expiration using
362      * the default expiration time (6 hours).
363      * For cache "PAGE", this method does not check expiration, and retrieves the data
364      * from the file from disk, NOT from the memory cache.
365      * @param poolname name of the cache pool, either "HENK" or "PAGE"
366      * @param res reponse object for retrieving headers (used by mmbase.org?)
367      * only needed for cachepool "PAGE"
368      * @param key URL of the page to store
369      * @param value the page content to store
370      * @param mimeType the page's mime type, only needed for cachepool "PAGE"
371      * @return the page's old content as a string, or <code>null</code> if no entry was found
372      * (i.e. cache was empty or poolname was invalid).
373      */
374     public String newput(String poolName,HttpServletResponse res, String key,String value, String mimeType) {
375         if (status==false) return null; // no caching if inactive
376 LRUHashtable pool=(LRUHashtable)pools.get(poolName);
377         if (pool==null) {
378             // create a new pool
379 pool=new LRUHashtable();
380             pools.put(poolName,pool);
381         }
382         // insert the new item and save to disk
383 if (poolName.equals("HENK")) {
384             saveFile(poolName,key,value);
385             timepool.put(poolName+key,new Integer ((int)(System.currentTimeMillis()/1000))); // store expiration time
386 return (String )pool.put(key,value);
387         } else if (poolName.equals("PAGE")) {
388             saveFile(poolName,key,value);
389             // new file for asis support
390 int pos=key.indexOf('?');
391             String filename=key;
392             if (pos!=-1) {
393                 filename=filename.replace('?',':');
394                 filename+=".asis";
395             } else {
396                 filename+=":.asis";
397             }
398             // obtain and add headers
399 // ----------------------
400 // org.mmbase String body="Status:"+(((worker)res).getWriteHeaders()).substring(8);
401 String body=getWriteHeaders(value, mimeType);
402             body+=value;
403             saveFile(poolName,filename,body);
404             // signal to start transfer of file to mirrors
405 signalNetFileSrv(filename);
406             return (String )pool.put(key,value);
407         }
408         log.error("newPut("+poolName+",HttpServletRequest,"+key+","+value+"): poolname("+poolName+") is not a valid cache name!");
409         return null;
410     }
411
412     /**
413      * Store a file in the indicated pool's cache (both in file and in the memory cache).
414      * Returns the old value if available.
415      * Is used in scanpage to recalculate the cached page.
416      * @param poolname name of the cache pool, either "HENK" or "PAGE"
417      * @param key URL of the page to store
418      * @param value the page content to store
419      * @param int cachetype only needed for cachepool "PAGE".
420      * If 0, no file transfer is performed. Otherwise the {@link NetFileSrv} builder is
421      * invoked to start the VWM that handles the transfer.
422      * @param mimeType the page's mime type, only needed for cachepool "PAGE"
423      * @return the page's old content as a string, or <code>null</code> if no entry was found
424      * (i.e. cache was empty or poolname was invalid).
425      */
426     public String newput2(String poolName,String key,String value,int cachetype, String mimeType) {
427         if (status==false) return null; // no caching when inactive
428 LRUHashtable pool=(LRUHashtable)pools.get(poolName);
429         if (pool==null) {
430             // create a new pool
431 pool=new LRUHashtable();
432             pools.put(poolName,pool);
433         }
434         log.debug("newput2("+poolName+","+key+","+value+","+cachetype+"): NEWPUT");
435         // insert the new item and save to disk
436 // XXX (why not call put ?)
437 if (poolName.equals("HENK")) {
438             saveFile(poolName,key,value);
439             // also add time to timepool??
440 timepool.put(poolName+key,new Integer ((int)(System.currentTimeMillis()/1000))); // store expiration time
441 return (String )pool.put(key,value);
442         } else if (poolName.equals("PAGE")) {
443             saveFile(poolName,key,value);
444             // new file for asis support
445 // -------------------------
446 int pos=key.indexOf('?');
447             String filename=key;
448             if (pos!=-1) {
449                 filename=filename.replace('?',':');
450                 filename+=".asis";
451             } else {
452                 filename+=":.asis";
453             }
454             // obtain and add headers
455 String body=getWriteHeaders(value, mimeType);
456             body+=value;
457             log.debug("newput2("+poolName+","+key+","+value+","+cachetype+"): NEWPUT=SAVE");
458             saveFile(poolName,filename,body);
459             if (cachetype!=0) signalNetFileSrv(filename);
460             return (String )pool.put(key,value);
461         }
462         log.error("newput2("+poolName+","+key+","+value+","+cachetype+"): poolName("+poolName+") is not a valid cachetype!");
463         return null;
464     }
465
466     /**
467      * Store a file in the indicated pool's cache (both on file and in the memory cache).
468      * Returns the old value if available.
469      * @param poolname name of the cache pool
470      * @param key URL of the page to store
471      * @param value the page content to store
472      * @return the page's old content as a string, or <code>null</code> if no entry was found
473      */
474     public String put(String poolName, String key,String value) {
475         if (status==false) return null; // no caching if inactive
476 LRUHashtable pool=(LRUHashtable)pools.get(poolName);
477         if (pool==null) {
478             // create a new pool
479 pool=new LRUHashtable();
480             pools.put(poolName,pool);
481         }
482         // got pool now insert the new item and save to disk
483 saveFile(poolName,key,value);
484
485         return (String )pool.put(key,value);
486     }
487
488     public Hashtable state() {
489         /*
490         state.put("Hits",""+hits);
491         state.put("Misses",""+miss);
492         */
493         return state;
494     }
495
496     /**
497      * Retrieve a description of the module's function.
498      */
499     public String getModuleInfo() {
500         return "This module provides cache functionality for text pages";
501     }
502
503     /**
504      * Saves a file to disk.
505      * The file is stored under the cache cache root directory, followed by the poolname
506      * (HENK or PAGE), followed by the 'original' file name.
507      * @param pool The name of the pool
508      * @param filename the name of the file
509      * @param value the value to store in the file
510      */
511     private boolean saveFile(String pool,String filename,String value) {
512         log.debug("saveFile("+pool+","+filename+",length("+value.length()+" bytes): saving!");
513         File sfile = new File(cachepath+pool+filename);
514         try {
515             DataOutputStream scan = new DataOutputStream(new FileOutputStream(sfile));
516             scan.writeBytes(value);
517             scan.flush();
518             scan.close();
519         } catch(Exception e) {
520             // make dirs only when an exception occurs... argh
521 // e.printStackTrace();
522 String dname=cachepath+pool+filename;
523             int pos=dname.lastIndexOf('/');
524             String dirname=dname.substring(0,pos);
525             File file = new File(dirname);
526             try {
527                 if (file.mkdirs()) {
528                     DataOutputStream scan = new DataOutputStream(new FileOutputStream(sfile));
529                     scan.writeBytes(value);
530                     scan.flush();
531                     scan.close();
532                 } else {
533                     log.error("scandisk cache -> making "+dirname+" failed ");
534                 }
535             } catch (Exception f) {
536                     log.error("scandisk cache -> Saving file "+filename+" failed "+f);
537             }
538             return false;
539         }
540         return true;
541     }
542
543     /**
544      * loads a file from the disk.
545      * The file retrieved is stored in the cache root directory, followed by the poolname
546      * (HENK or PAGE), followed by the 'original' file name.
547      * @param pool The name of the pool
548      * @param filename the name of the file
549      * @return the content of the file in a {@link fileInfo} object.
550      */
551     public fileInfo loadFile(String pool,String filename) {
552         fileInfo fileinfo=new fileInfo();
553         try {
554             File sfile = new File(cachepath+pool+filename);
555             FileInputStream scan =new FileInputStream(sfile);
556             int filesize = (int)sfile.length();
557             byte[] buffer=new byte[filesize];
558             int len=scan.read(buffer,0,filesize);
559             if (len!=-1) {
560                 String value=new String (buffer,0);
561                 fileinfo.value=value;
562                 fileinfo.time=(int)(sfile.lastModified()/1000);
563                                 log.debug("loadFile last modified " + sfile.lastModified()/1000);
564                 return fileinfo;
565             }
566             scan.close();
567         } catch(Exception e) {
568             // e.printStackTrace();
569 return null;
570         }
571         return null;
572     }
573
574     /**
575      * Signal the NetFileServ builder that the .asis file for a page has changed.
576      * The builder then searches the responsible VWM that handles the mirrorring of the pages,
577      * and activates it.
578      * @param filename the .asis filename that changed
579      */
580     public void signalNetFileSrv(String filename) {
581         log.debug("signalNetFileSrv("+filename+"): SIGNAL");
582         if (mmbase!=null) {
583             NetFileSrv bul=(NetFileSrv)mmbase.getMMObject("netfilesrv");
584             if (bul!=null) {
585                 ((NetFileSrv)bul).fileChange("pages","main",filename);
586             }
587         } else {
588             log.error("signalNetFileSrv("+filename+"): can't use NetFileSrv builder");
589         }
590     }
591
592     /**
593      * Returns the headers for a .asis file to be stored for a "PAGE" cache.
594      * @param value page content, used to set the Content-length header.
595      * @param mimeType the mimetype of the page. default (if unspecified) is text/html; iso-8859-1
596      * @return the page headers as a <code>String</code>
597      */
598     String getWriteHeaders(String value, String mimeType) {
599         if ((mimeType==null) || mimeType.equals("") || mimeType.equals("text/html"))
600             mimeType = "text/html; charset=\"iso-8859-1\"";
601         String now = RFC1123.makeDate(new Date (System.currentTimeMillis()));
602         String expireTime = RFC1123.makeDate(new Date (System.currentTimeMillis()+15000));
603         String body="Status: 200 OK\n";
604         body+="Server: OrionCache\n"; // orion cache ???
605 body+="Content-type: "+mimeType+"\n";
606         body+="Content-length: "+value.length()+"\n";
607         body+="Expires: "+expireTime+"\n";
608         body+="Date: "+now+"\n";
609                 // Internet explorer refuses to see resulting page as HTML
610 // when cache control header added to .asis file
611 // body+="Cache-Control: no-cache\n";
612 body+="Pragma: no-cache\n";
613         body+="Last-Modified: "+now+"\n\n";
614         return body;
615     }
616
617     /**
618      * Removes an entry from the cache pool (both the file on disk and in the memory cache).
619      * If the pool is "PAGE", the file will only be removed from the local cache,
620      * not from any mirror servers.
621      * @param pool name of cache pool, expected (but not verified) "HENK" or "PAGE"
622      * @param key URL of the page to remove
623      */
624     public void remove(String poolName, String key) {
625             File file = new File(cachepath + poolName + key);
626             file.delete();
627             LRUHashtable pool=(LRUHashtable)pools.get(poolName);
628             if (pool!=null) pool.remove(key);
629             timepool.remove(poolName + key);
630     }
631
632     /**
633      * Returns the status of this module.
634      * @return <code>true</code> if the module is active, <code>false</code> otherwise
635      */
636     public boolean getStatus() {
637         return status;
638     }
639
640         /**
641          * This method signals the scanparser to start caclulation on the page
642          * given in uri/scanpage, it will duplicate the request as not to interfere
643          * with the original request.
644          * @param sp The original requests scanpage
645          * @param uri of the request
646          */
647         private void signalProcessor(scanpage sp, String uri) {
648                 scanpage fakesp=sp.duplicate();
649                 scanparser.processPage(fakesp,uri);
650         }
651 }
652

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags