ImageMaster


1   /*
2   
3   This software is OSI Certified Open Source Software.
4   OSI Certified is a certification mark of the Open Source Initiative.
5   
6   The license (Mozilla version 1.0) can be read at the MMBase site.
7   See http://www.MMBase.org/license
8   
9   */
10  
11  package org.mmbase.module.builders.vwms;
12  
13  import java.util.*;
14  import java.io.*;
15  
16  import org.mmbase.module.core.*;
17  import org.mmbase.util.*;
18  import org.mmbase.module.builders.*;
19  import org.mmbase.util.images.Imaging;
20  import org.mmbase.util.logging.*;
21  
22  /**
23   * A VWM that manages image files by scheduling them to be send to one or more mirror sites.
24   * Requests for scheduling is done in the netfile builder.
25   * This VWM handles those netfile requests whose service is 'images'. Available subservices are 'main' and 'mirror'.
26   * Requests for file copy are checked periodically. The result is one or more requests for a 'mirror' service,
27   * which then result in a file copy request, which is handled in a separate thread.
28   * Before copying, images are retrieved from Icache and converted to an 'asis' file
29   * (this is the file that actually gets copied).
30   *
31   * @author Daniel Ockeloen
32   * @author Pierre van Rooden (javadocs)
33   * @version $Id: ImageMaster.java,v 1.28 2005/11/23 15:45:13 pierre Exp $
34   */
35  
36  public class ImageMaster extends Vwm implements MMBaseObserver,VwmServiceInterface {
37  
38      // Logger
39      private static Logger log = Logging.getLoggerInstance(ImageMaster.class.getName());
40  
41      Hashtable properties;
42  
43      // field used to skip first probeCall (why???)
44      boolean first=true;
45  
46      Object   syncobj=new Object  ();  // used in commented code
47  
48      /**
49       * List of files to transfer.
50       * The filelist is periodically cleared by {@link ImagePusher} (which purges duplicate
51       * files and handles the remaining transfers).
52       */
53      Vector files=new Vector();
54      /**
55       * The background thread that takes care of of the files scheduled for transfer.
56       */
57      ImagePusher pusher;
58      /**
59       * The maximum number of 'main' or 'mirror' requests to handle during each {@link #probeCall}
60       */
61      private int maxSweep=16;
62  
63      /**
64       * Constructor for ImageMaster
65       */
66      public ImageMaster() {
67          log.info("VWM ImageMaster started");
68      }
69  
70      /**
71       * Performs general periodic maintenance.
72       * This routine handles alle open images/main and images/mirror file service requests.
73       * These requests are obtained from the netfiles builder.
74       * For each file that should be serviced, the filechange method is called.
75       * This routine handles a maximum of 10 page/main, and 50 page/mirror service
76       * calls each time it is called.
77       * The first time this method is call, nothing happens (?)
78       * <br />
79       * Very similar to {@link #probeCall}.
80       *
81       * @return <code>true</code> if maintenance was performed, <code>false</code> otherwise
82       */
83      public boolean probeCall() {
84          if (first) {
85              first=false;
86              // create ImagePusher
87              if (pusher==null) {
88                  pusher=new ImagePusher(this);
89                  log.info("ImageMaster -> Starting Image pusher");
90              }
91          } else {
92              try {
93                  Netfiles bul=(Netfiles)Vwms.getMMBase().getMMObject("netfiles");
94                  // note: order Descending means last file is transferred first.
95                  // Theoretically, some files may never be handled due to use of maxsweep
96                  Enumeration e=bul.search("WHERE service='images' AND subservice='main' AND status="+Netfiles.STATUS_REQUEST+" ORDER BY number DESC");
97                  int i=0;
98                  while (e.hasMoreElements() && i<maxSweep) {
99                      MMObjectNode node=(MMObjectNode)e.nextElement();
100                     fileChange(""+node.getIntValue("number"),"c");
101                     i++;
102                 }
103                 try { Thread.sleep(1500); } catch(InterruptedException   x) {}
104                 Enumeration f=bul.search("WHERE service='images' AND subservice='mirror' AND status="+Netfiles.STATUS_REQUEST+" ORDER BY number DESC");
105                 i=0;
106                 while (f.hasMoreElements() && i<maxSweep) {
107                     MMObjectNode node=(MMObjectNode)f.nextElement();
108                     fileChange(""+node.getIntValue("number"),"c");
109                     i++;
110                 }
111             } catch(Exception   e) {
112                 log.error("probeCall exception "+e);
113                 log.error(Logging.stackTrace(e));
114             }
115         }
116         return true;
117     }
118 
119     /**
120      * Called when a remote node is changed.
121          * @param machine Name of the machine that changed the node.
122      * @param number Number of the changed node as a <code>String</code>
123      * @param builder type of the changed node
124      * @param ctype command type, 'c'=changed, 'd'=deleted', 'r'=relations changed, 'n'=new
125      * @return <code>true</code>
126      */
127     public boolean nodeRemoteChanged(String   machine,String   number,String   builder,String   ctype) {
128         return(nodeChanged(machine,number,builder,ctype));
129     }
130 
131     /**
132      * Called when a local node is changed.
133          * @param machine Name of the machine that changed the node.
134      * @param number Number of the changed node as a <code>String</code>
135      * @param builder type of the changed node
136      * @param ctype command type, 'c'=changed, 'd'=deleted', 'r'=relations changed, 'n'=new
137      * @return <code>true</code>
138      */
139     public boolean nodeLocalChanged(String   machine,String   number,String   builder,String   ctype) {
140         return nodeChanged(machine,number,builder,ctype);
141     }
142 
143     /**
144      * Called when a local or remote node is changed.
145      * Does not take any action.
146          * @param machine Name of the machine that changed the node.
147      * @param number Number of the changed node as a <code>String</code>
148      * @param builder type of the changed node
149      * @param ctype command type, 'c'=changed, 'd'=deleted', 'r'=relations changed, 'n'=new
150      * @return <code>true</code>
151      */
152     public boolean nodeChanged(String   machine,String   number,String   builder, String   ctype) {
153         log.debug("sees that : "+number+" has changed type="+ctype+" of type:"+builder+" by machine:"+machine);
154         return true;
155     }
156 
157     /**
158      * Schedules a service-request on a file.
159      * Only "images/main" services are handled.
160      * The service-request is later handled through the {@link #probeCall} method.
161      * @param service the service to be performed
162      * @param subservice the subservice to be performed
163      * @param filename the filename to service
164      * @return <code>true</code> if maintenance was performed, <code>false</code> otherwise
165      */
166     public boolean fileChange(String   service,String   subservice,String   filename) {
167         filename=URLEscape.unescapeurl(filename);
168         log.debug("fileChange -> "+filename);
169         // jump to correct subhandles based on the subservice
170         if (subservice.equals("main")) {
171             handleMainCheck(service,subservice,filename);
172         }
173         return true;
174     }
175 
176     /**
177      * Handles a service-request on a file, registered in the netfiles builder.
178      * Depending on the subservice requested, this routine calls {@link #handleMirror}
179      * or {@link #handleMain}.
180      * @param number Number of the node in the netfiles buidler than contain service request information.
181      * @param ctype the type of change on that node ("c" : node was changed)
182      * @return <code>true</code>
183      */
184     public boolean fileChange(String   number, String   ctype) {
185         // debug("fileChange="+number+" "+ctype);
186         // first get the change node so we can see what is the matter with it.
187         Netfiles bul=(Netfiles)Vwms.getMMBase().getMMObject("netfiles");
188         MMObjectNode filenode=bul.getNode(number);
189         if (filenode!=null) {
190             // obtain all the basic info on the file.
191             String   service=filenode.getStringValue("service");
192             String   subservice=filenode.getStringValue("subservice");
193             int status=filenode.getIntValue("status");
194 
195             log.debug("fileChange "+number+" "+subservice+" "+status);
196 
197             // jump to correct subhandles based on the subservice
198             if (subservice.equals("main")) {
199                 handleMain(filenode,status,ctype);
200             } else if (subservice.equals("mirror")) {
201                 handleMirror(filenode,status,ctype);
202             }
203         }
204         return(true);
205     }
206 
207     /**
208      * Return a @link{ ByteFieldContainer} containing the bytes and object number
209      * for the cached image with a certain ckey, or null, if not cached.
210      * @param ckey teh ckey to search for
211      * @return null, or a @link{ ByteFieldContainer} object
212      */
213     public ByteFieldContainer getCkeyNode(ImageCaches bul, String   ckey) {
214         log.debug("getting ckey node with " + ckey);
215         int pos = 0;
216         while (Character.isDigit(ckey.charAt(pos))) pos ++;
217         int nodeNumber = Integer.parseInt(ckey.substring(0, pos));
218         String   template   = ckey.substring(pos);
219         if (template.charAt(0) == '=') template = template.substring(1);
220         MMObjectNode node = bul.getCachedNode(nodeNumber, template);
221         if (node == null) {
222             // we dont have a cachednode yet, return null
223             log.debug("cached node not found for key (" + ckey + "), returning  null");
224             return null;
225         }
226         // find binary data
227         byte data[] = node.getByteValue(Imaging.FIELD_HANDLE);
228         if (data == null) {
229             // if it didn't work, also cache this result, to avoid concluding that again..
230             // should this trow an exception every time? I think so, otherwise  we would generate an
231             // image every time it is requested, which also net very handy...
232             String   msg =
233                 "The node(#" + node.getNumber() + ") which should contain the cached result for ckey:" + ckey +
234                 " had as value <null>, this means that something is really wro ng.(how can we have an cache node with node value in it?)";
235             log.error(msg);
236             throw new RuntimeException  (msg);
237          }
238         ByteFieldContainer result = new ByteFieldContainer(node.getNumber(), data);
239         return result;
240     }
241 
242     /**
243      * Handles an images/mirror service request.
244      * Converts images to an asis file format, then places the asis file in the files list,
245      * so it will be sent to a mirror site by the ImagePusher.
246      * @param node the filenet node that contains the service request
247      * @param status the current status of the node
248      * @param ctype the type of change on that node ("c" : node was changed)
249      * @return <code>true</code>
250      */
251     public boolean handleMirror(MMObjectNode filenode,int status,String   ctype) {
252         if (filenode==null) {
253             log.error("ERROR: handleMirror filenode null!");
254             return true;
255         }
256 
257         log.debug("Node "+filenode+" status "+status+" type "+ctype);
258         switch(status) {
259             case Netfiles.STATUS_REQUEST:  // Request
260                 log.debug("status=="+Netfiles.STATUS_REQUEST);
261                 filenode.setValue("status",Netfiles.STATUS_ON_ITS_WAY);
262                 filenode.commit();
263                 log.debug("Starting real work");
264                 // do stuff
265                 String   filename=filenode.getStringValue("filename");
266                 if ((filename==null) || filename.equals("")) {
267                     log.error("handleMirror: filename null");
268                     return true;
269                 }
270                 log.debug("handleMirror"+filename);
271                 String   dstserver=filenode.getStringValue("mmserver");
272 
273                 // save the image to disk
274                 ImageCaches bul=(ImageCaches)Vwms.getMMBase().getMMObject("icaches");
275                 if (bul==null) {
276                     log.error("ImageCaches builder is null");
277                     return true;
278                 }
279 
280                 String   mimetype = "image/jpeg"; // When not overwritten, it will stay on 'jpeg'.
281 
282                 // get the clear ckey
283                 // '/img.db?xxxxxxxxxx.asis'
284                 // zap '/img.db?' and '.asis'
285                 String   ckey=filename.substring(8);
286                 int pos=ckey.lastIndexOf(".");
287                 if (pos!=-1) {
288                     ckey=ckey.substring(0,pos);
289                     // We now have a clean ckey ( aka 234242+f(gif) )
290                     // Get mimetype from ckey params string.
291                     StringTokenizer st = new StringTokenizer(ckey,"+\n\r");
292                     Vector ckeyVec = new Vector();
293                     while (st.hasMoreTokens()) {
294                         ckeyVec.addElement(st.nextElement());
295                     }
296                     Images imagesBuilder = (Images)Vwms.getMMBase().getMMObject("images");
297                     if (imagesBuilder==null) {
298                         log.error("handleMirror images builder not found");
299                         return true;
300                     }
301                     mimetype = getImageMimeType(imagesBuilder, ckeyVec);
302                     // debug("handleMirror: ckey "+ckey+" has mimetype: "+mimetype);
303                     ckey=path2ckey(ckey, imagesBuilder);
304                 }
305 
306                 log.debug("handleMirror: ckey "+ckey);
307                 ByteFieldContainer container = getCkeyNode(bul, ckey);
308                 if (container == null) {
309                     log.debug("handleMirror: no icaches entry yet");
310                 }
311                 byte[] filebuf = container.value;
312                 log.debug("request size "+filebuf.length);
313                 String   srcpath=getProperty("test1:path"); // ??? XXX should be changed!
314                 // Pass mimetype. should check on succes sor failure!
315                 saveImageAsisFile(srcpath,filename,filebuf,mimetype);
316 
317                 // recover the correct source/dest properties for this mirror
318                 String   sshpath=getProperty("sshpath");
319                 String   dstuser=getProperty(dstserver+":user");
320                 String   dsthost=getProperty(dstserver+":host");
321                 String   dstpath=getProperty(dstserver+":path");
322 
323 /*
324                 SCPcopy scpcopy=new SCPcopy(sshpath,dstuser,dsthost,dstpath);
325 
326                 synchronized(syncobj) {
327                     scpcopy.copy(srcpath,filename);
328                 }
329 */
330                 // should be synchronized
331                 files.addElement(new aFile2Copy(dstuser,dsthost,dstpath,srcpath,filename,sshpath));
332 
333                 // remove the tmp image file
334 
335                 filenode.setValue("status",Netfiles.STATUS_DONE);
336                 filenode.commit();
337                 break;
338             case Netfiles.STATUS_ON_ITS_WAY:  // On Its Way
339                 break;
340             case Netfiles.STATUS_DONE:  // Done
341                 break;
342             default:
343                 log.error("This cannot happen, contact your system administrator");
344                 break;
345         }
346         return true;
347     }
348 
349     /**
350      * Handles a images/main service request.
351      * Schedules requests to mirror the file using {@link #doMainRequest}<br />
352      * @param node the filenet node that contains the service request
353      * @param status the current status of the node
354      * @param ctype the type of change on that node ("c" : node was changed)
355      * @return <code>true</code>
356      */
357     public boolean handleMain(MMObjectNode filenode,int status,String   ctype) {
358         switch(status) {
359             case Netfiles.STATUS_REQUEST:  // Request
360                 filenode.setValue("status",Netfiles.STATUS_ON_ITS_WAY);
361                 filenode.commit();
362                 // do stuff
363                 doMainRequest(filenode);
364                 filenode.setValue("status",Netfiles.STATUS_DONE);
365                 filenode.commit();
366                 break;
367             case Netfiles.STATUS_ON_ITS_WAY:  // On Its Way
368                 break;
369             case Netfiles.STATUS_DONE:  // Done
370                 break;
371             default:
372                 log.error("This cannot happen, contact your system administrator");
373                 break;
374         }
375         return true;
376     }
377 
378     /**
379      * Handles a main subservice on an image.
380      * The image is scheduled to be sent to all appropriate mirrorsites for this service,
381      * by setting the request status in the associated mirror nodes.
382      * If no mirror nodes are associated with this page, nothing happens.
383      * @param filenode the netfiles node with the original (main) request
384      */
385     public boolean doMainRequest(MMObjectNode filenode) {
386         log.debug("doMainRequest for "+filenode.getIntValue("number")+" "+filenode.getStringValue("filename"));
387         // so this file has changed probably, check if the file is ready on
388         // disk and set the mirrors to dirty/request.
389         String   filename = filenode.getStringValue("filename");
390         String   service = filenode.getStringValue("service");
391 
392         // find and change all the mirror node so they get resend
393         Netfiles bul=(Netfiles)Vwms.getMMBase().getMMObject("netfiles");
394         Enumeration e=bul.search("WHERE filename='"+filename+"' AND service='"+service+"' AND subservice='mirror'");
395         if (!e.hasMoreElements()) {
396             log.debug("doMainRequest: No mirror nodes found for : "+filenode.toString()+" !!");
397         }
398         while (e.hasMoreElements()) {
399             MMObjectNode mirrornode=(MMObjectNode)e.nextElement();
400             log.debug("doMainRequest sending change for "+mirrornode.getIntValue("number"));
401             mirrornode.setValue("status",Netfiles.STATUS_REQUEST);
402             mirrornode.commit();
403         }
404         return true;
405     }
406 
407     /**
408      * Schedules a netfile object to be send to its mirror sites.
409      * The routine searches the appropriate netfile node, and sets its status to 'request'.
410      * If a node does not exits, a new node is created. In the latter case, the system also creates mirrornodes
411      * for each mirrorsite associated with this service. (actually, it creates one mirrornode for a vpro-server,
412      * but this should be altered).
413      * @param service the service to be performed
414      * @param subservice the subservice to be performed
415      * @param filename the filename to service
416      */
417     public synchronized void handleMainCheck(String   service,String   subservice,String   filename) {
418         Netfiles bul=(Netfiles)Vwms.getMMBase().getMMObject("netfiles");
419         Enumeration e=bul.search("WHERE filename='"+filename+"' AND service='"+service+"' AND subservice='"+subservice+"'");
420         if (e.hasMoreElements()) {
421             log.debug("handleMainCheck: existing file");
422             MMObjectNode mainnode=(MMObjectNode)e.nextElement();
423             int currentstatus=mainnode.getIntValue("status");
424             if (currentstatus>Netfiles.STATUS_ON_ITS_WAY) { // check only the ones that are done
425                 mainnode.setValue("status",Netfiles.STATUS_REQUEST);
426                 mainnode.commit();
427             }
428         } else {
429             log.debug("handleMainCheck: new file");
430             MMObjectNode mainnode=bul.getNewNode("system");
431             mainnode.setValue("filename",filename);
432             mainnode.setValue("mmserver","test1"); // eeks!
433             mainnode.setValue("service",service);
434             mainnode.setValue("subservice",subservice);
435             mainnode.setValue("status",Netfiles.STATUS_DONE);
436             mainnode.setValue("filesize",-1);
437             bul.insert("system",mainnode);
438 
439             // create mirror nodes
440             // should use same getMirrorNodes() system as PageMaster
441             mainnode=bul.getNewNode("system");
442             mainnode.setValue("filename",filename);
443             mainnode.setValue("mmserver","omroep"); // aaaaaarghgh
444             mainnode.setValue("service",service);
445             mainnode.setValue("subservice","mirror");
446             mainnode.setValue("status",Netfiles.STATUS_REQUEST);
447             mainnode.setValue("filesize",-1);
448             bul.insert("system",mainnode);
449         }
450     }
451 
452     /**
453      * Retrieves a named property of a server.
454      * Should use the same system as PageMaster (retrieve data from MSMerver).
455      * @param machine name of the server
456      * @param key name of the property to retrieve
457      * @return the property value
458      */
459     public String   getProperty(String   key) {
460         if (properties==null) initProperties();
461         return (String  )properties.get(key);
462     }
463 
464     /**
465      * Initializes server properties.
466      * @deprecated (vpro specific code)
467      */
468     private void initProperties() {
469         properties=new Hashtable();
470         properties.put("sshpath","/usr/local/bin");
471         properties.put("omroep:user","vpro");
472         properties.put("omroep:host","vpro.omroep.nl");
473         properties.put("omroep:path","/bigdisk/htdocs/");
474         properties.put("test1:path","/usr/local/log/james/scancache/PAGE");
475     }
476 
477     /**
478      * Stores an array byte (presumably an image) as a asis file.
479      * @param path path of the asis file
480      * @param filename name of the asis file
481      * @param value the bytearray to store
482      * @param mimetype mimetype of the byte array, i.e. image/jpeg
483      * @return <code>true</code> (what if it fails??)
484      */
485     private boolean saveImageAsisFile(String   path,String   filename,byte[] value, String   mimetype) {
486         String   header="Status: 200 OK";
487         // header+="\r\nContent-type: image/jpeg";
488         header+="\r\nContent-type: "+mimetype;
489         header+="\r\nContent-length: "+value.length;
490         header+="\r\n\r\n";
491 
492         File sfile = new File(path+filename);
493         try {
494             DataOutputStream scan = new DataOutputStream(new FileOutputStream(sfile));
495             scan.writeBytes(header);
496             scan.write(value);
497             scan.flush();
498             scan.close();
499         } catch(Exception   e) {
500             log.error(e.getMessage());
501             log.error(Logging.stackTrace(e));
502         }
503         return true;
504     }
505 
506     /**
507      * Converts an asis filename to a key that can be used to retrieve an Icache node
508      * @param path the asis fielname
509      * @param imageBuilder Reference to the Images builder, unused
510      * @return an appriopriate Icache key
511      */
512     private String   path2ckey(String   path, Images imageBuilder) {
513         StringTokenizer tok = new StringTokenizer(path,"+\n\r");
514         String   ckey=tok.nextToken();
515         //ckey = ""+imageBuilder.convertAlias(ckey);
516         while (tok.hasMoreTokens()) {
517             String   key=tok.nextToken();
518             ckey+=key;
519         }
520         return ckey;
521     }
522 
523     /**
524     * Will return {@link #defaultImageType} as default type, or one of the strings in params, must contain the following "f(type)" where type will be returned
525      * @param params a <code>List</code> of <code>String</code>s, which could contain the "f(type)" string
526      * @return {@link #defaultImageType} by default, or the first occurence of "f(type)"
527      *
528      *
529      */
530     private String   getImageMimeType(Images images, List params) {
531         String   format = null;
532         String   key;
533 
534         // WHY the itype colomn isn't used?
535 
536         for (Iterator e = params.iterator() ;e.hasNext();) {
537             key = (String  )e.next();
538 
539             // look if our string is long enough...
540             if(key != null && key.length() > 2) {
541                 // first look if we start with an "f("... format is f(gif)
542                 if(key.startsWith("f(")) {
543                     // one search function remaining...
544                     int pos = key.lastIndexOf(')');
545                     // we know for sure that our "(" is at pos 1, so we can define this hard...
546                     format = key.substring(2, pos);
547                     break;
548                 }
549             }
550         }
551         if (format == null) format = images.getDefaultImageType();
552         String   mimetype = Imaging.getMimeTypeByExtension(format);
553         if (log.isDebugEnabled()) {
554             log.debug("getImageMimeType: getMMBase().getMimeType(" + format + ") = " + mimetype);
555         }
556         return mimetype;
557     }
558 
559 }
560
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags