KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > archive > crawler > admin > ui > JobConfigureUtils


1 /*
2  * Heritrix
3  *
4  * $Id: JobConfigureUtils.java,v 1.21 2006/07/31 19:39:04 gojomo Exp $
5  *
6  * Created on Aug 30, 2004
7  *
8  * Copyright (C) 2003 Internet Archive.
9  *
10  * This file is part of the Heritrix web crawler (crawler.archive.org).
11  *
12  * Heritrix is free software; you can redistribute it and/or modify
13  * it under the terms of the GNU Lesser Public License as published by
14  * the Free Software Foundation; either version 2.1 of the License, or
15  * any later version.
16  *
17  * Heritrix is distributed in the hope that it will be useful,
18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20  * GNU Lesser Public License for more details.
21  *
22  * You should have received a copy of the GNU Lesser Public License
23  * along with Heritrix; if not, write to the Free Software
24  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25  */
26 package org.archive.crawler.admin.ui;
27
28 import java.io.BufferedReader JavaDoc;
29 import java.io.BufferedWriter JavaDoc;
30 import java.io.File JavaDoc;
31 import java.io.FileInputStream JavaDoc;
32 import java.io.FileWriter JavaDoc;
33 import java.io.IOException JavaDoc;
34 import java.io.InputStream JavaDoc;
35 import java.io.InputStreamReader JavaDoc;
36 import java.io.Reader JavaDoc;
37 import java.io.StringReader JavaDoc;
38 import java.io.Writer JavaDoc;
39 import java.lang.reflect.Constructor JavaDoc;
40 import java.lang.reflect.InvocationTargetException JavaDoc;
41 import java.util.logging.Level JavaDoc;
42 import java.util.logging.Logger JavaDoc;
43
44 import javax.management.Attribute JavaDoc;
45 import javax.management.AttributeNotFoundException JavaDoc;
46 import javax.management.InvalidAttributeValueException JavaDoc;
47 import javax.management.MBeanAttributeInfo JavaDoc;
48 import javax.management.MBeanException JavaDoc;
49 import javax.management.ReflectionException JavaDoc;
50 import javax.servlet.http.HttpServletRequest JavaDoc;
51 import javax.servlet.http.HttpServletResponse JavaDoc;
52
53 import org.archive.crawler.admin.CrawlJob;
54 import org.archive.crawler.admin.CrawlJobHandler;
55 import org.archive.crawler.settings.ComplexType;
56 import org.archive.crawler.settings.CrawlerSettings;
57 import org.archive.crawler.settings.ListType;
58 import org.archive.crawler.settings.MapType;
59 import org.archive.crawler.settings.ModuleAttributeInfo;
60 import org.archive.crawler.settings.ModuleType;
61 import org.archive.crawler.settings.SettingsHandler;
62 import org.archive.crawler.settings.XMLSettingsHandler;
63 import org.archive.crawler.settings.refinements.Refinement;
64 import org.archive.util.IoUtils;
65
66 /**
67  * Utility methods used configuring jobs in the admin UI.
68  *
69  * Methods are mostly called by the admin UI jsp.
70  *
71  * @author stack
72  * @version $Date: 2006/07/31 19:39:04 $, $Revision: 1.21 $
73  */
74 public class JobConfigureUtils {
75     private static Logger JavaDoc logger = Logger.getLogger(JobConfigureUtils.class
76             .getName());
77     public static final String JavaDoc ACTION = "action";
78     public static final String JavaDoc SUBACTION = "subaction";
79     public static final String JavaDoc FILTERS = "filters";
80     private static final String JavaDoc MAP = "map";
81     private static final String JavaDoc FILTER = "filter";
82     private static final Object JavaDoc ADD = "add";
83     private static final Object JavaDoc MOVEUP = "moveup";
84     private static final Object JavaDoc MOVEDOWN = "movedown";
85     private static final Object JavaDoc REMOVE = "remove";
86     private static final Object JavaDoc GOTO = "goto";
87     private static final Object JavaDoc DONE = "done";
88     private static final Object JavaDoc CONTINUE = "continue"; // keep editting
89
90     /**
91      * Check passed crawljob CrawlJob setting. Call this method at start of
92      * page.
93      *
94      * @param job
95      * Current CrawlJobHandler.
96      * @param request
97      * Http request.
98      * @param response
99      * Http response.
100      * @return Crawljob.
101      */
102     protected static CrawlJob getAndCheckJob(CrawlJob job,
103             HttpServletRequest JavaDoc request, HttpServletResponse JavaDoc response) {
104         return job;
105     }
106
107     /**
108      * This methods updates a ComplexType with information passed to it by a
109      * HttpServletRequest. It assumes that for every 'simple' type there is a
110      * corresponding parameter in the request. A recursive call will be made for
111      * any nested ComplexTypes. For each attribute it will check if the relevant
112      * override is set (name.override parameter equals 'true'). If so the
113      * attribute setting on the specified domain level (settings) will be
114      * rewritten. If it is not we well ensure that it isn't being overridden.
115      *
116      * @param mbean
117      * The ComplexType to update
118      * @param settings
119      * CrawlerSettings for the domain to override setting for. null
120      * denotes the global settings.
121      * @param request
122      * The HttpServletRequest to use to update the ComplexType
123      * @param expert
124      * if true expert settings will be updated, otherwise they will
125      * be ignored.
126      */
127     public static void writeNewOrderFile(ComplexType mbean,
128             CrawlerSettings settings, HttpServletRequest JavaDoc request, boolean expert) {
129         // If mbean is transient or a hidden expert setting.
130 if (mbean.isTransient() || (mbean.isExpertSetting() && expert == false)) {
131             return;
132         }
133
134         MBeanAttributeInfo JavaDoc a[] = mbean.getMBeanInfo(settings).getAttributes();
135         for (int n = 0; n < a.length; n++) {
136             checkAttribute((ModuleAttributeInfo) a[n], mbean, settings,
137                     request, expert);
138         }
139     }
140
141     /**
142      * Process passed attribute. Check if needs to be written and if so, write
143      * it.
144      *
145      * @param att
146      * Attribute to process.
147      * @param mbean
148      * The ComplexType to update
149      * @param settings
150      * CrawlerSettings for the domain to override setting for. null
151      * denotes the global settings.
152      * @param request
153      * The HttpServletRequest to use to update the ComplexType
154      * @param expert
155      * if true expert settings will be updated, otherwise they will
156      * be ignored.
157      */
158     protected static void checkAttribute(ModuleAttributeInfo att,
159             ComplexType mbean, CrawlerSettings settings,
160             HttpServletRequest JavaDoc request, boolean expert) {
161         // The attributes of the current attribute.
162 Object JavaDoc currentAttribute = null;
163         try {
164             currentAttribute = mbean.getAttribute(settings, att.getName());
165         } catch (Exception JavaDoc e) {
166             logger.severe("Failed getting " + mbean.getAbsoluteName()
167                     + " attribute " + att.getName() + ": " + e.getMessage());
168             return;
169         }
170
171         if (logger.isLoggable(Level.FINE)) {
172             logger.fine("MBEAN: " + mbean.getAbsoluteName() + " "
173                     + att.getName() + " TRANSIENT " + att.isTransient() + " "
174                     + att.isExpertSetting() + " " + expert);
175         }
176
177         if (att.isTransient() == false
178                 && (att.isExpertSetting() == false || expert)) {
179             if (currentAttribute instanceof ComplexType) {
180                 writeNewOrderFile((ComplexType) currentAttribute, settings,
181                         request, expert);
182             } else {
183                 String JavaDoc attName = att.getName();
184                 // Have a 'setting'. Let's see if we need to update it (if
185 // settings == null update all, otherwise only if override
186 // is set.
187 String JavaDoc attAbsoluteName = mbean.getAbsoluteName() + "/"
188                         + attName;
189                 boolean override = (request.getParameter(attAbsoluteName
190                         + ".override") != null)
191                         && (request.getParameter(attAbsoluteName + ".override")
192                                 .equals("true"));
193                 if (settings == null || override) {
194                     if (currentAttribute instanceof ListType) {
195                         try {
196                             ListType list = (ListType)currentAttribute;
197                             Class JavaDoc cls = list.getClass();
198                             Constructor JavaDoc constructor = cls.getConstructor(String JavaDoc.class, String JavaDoc.class);
199                             list = (ListType) constructor.newInstance(list.getName(), list.getDescription());
200                             String JavaDoc[] elems = request
201                                     .getParameterValues(attAbsoluteName);
202                             for (int i = 0; elems != null && i < elems.length; i++) {
203                                 list.add(elems[i]);
204                             }
205                             writeAttribute(attName, attAbsoluteName, mbean,
206                                     settings, list);
207                         } catch (Exception JavaDoc e) {
208                             e.printStackTrace();
209                             logger.severe("Setting new list values on "
210                                     + attAbsoluteName + ": " + e.getMessage());
211                             return;
212                         }
213                     } else {
214                         writeAttribute(attName, attAbsoluteName, mbean,
215                                 settings, request.getParameter(attAbsoluteName));
216                     }
217
218                 } else if (settings != null && override == false) {
219                     // Is not being overridden. Need to remove possible
220 // previous overrides.
221 try {
222                         mbean.unsetAttribute(settings, attName);
223                     } catch (Exception JavaDoc e) {
224                         e.printStackTrace();
225                         logger.severe("Unsetting attribute on "
226                                 + attAbsoluteName + ": " + e.getMessage());
227                         return;
228                     }
229                 }
230             }
231         }
232     }
233
234     /**
235      * Write out attribute.
236      *
237      * @param attName
238      * Attribute short name.
239      * @param attAbsoluteName
240      * Attribute full name.
241      * @param mbean
242      * The ComplexType to update
243      * @param settings
244      * CrawlerSettings for the domain to override setting for. null
245      * denotes the global settings.
246      * @param value
247      * Value to set into the attribute.
248      */
249     protected static void writeAttribute(String JavaDoc attName,
250             String JavaDoc attAbsoluteName, ComplexType mbean,
251             CrawlerSettings settings, Object JavaDoc value) {
252         try {
253             if (logger.isLoggable(Level.FINE)) {
254                 logger.fine("MBEAN SET: " + attAbsoluteName + " " + value);
255             }
256             mbean.setAttribute(settings, new Attribute JavaDoc(attName, value));
257         } catch (Exception JavaDoc e) {
258             e.printStackTrace();
259             logger.severe("Setting attribute value " + value + " on "
260                     + attAbsoluteName + ": " + e.getMessage());
261             return;
262         }
263     }
264
265     /**
266      * Check passed job is not null and not readonly.
267      * @param job Job to check.
268      * @param response Http response.
269      * @param redirectBasePath Full path for where to go next if an error.
270      * @param currDomain May be null.
271      * E.g. "/admin/jobs/per/overview.jsp".
272      * @return A job else we've redirected if no job or readonly.
273      * @throws IOException
274      */
275     public static CrawlJob checkCrawlJob(CrawlJob job,
276         HttpServletResponse JavaDoc response, String JavaDoc redirectBasePath,
277         String JavaDoc currDomain)
278     throws IOException JavaDoc {
279         if (job == null) {
280             // Didn't find any job with the given UID or no UID given.
281 response.sendRedirect(redirectBasePath +
282                 "?message=No job selected");
283         } else if (job.isReadOnly()) {
284             // Can't edit this job.
285 response.sendRedirect(redirectBasePath +
286                 "?job=" + job.getUID() +
287                 ((currDomain != null && currDomain.length() > 0)?
288                     "&currDomain=" + currDomain: "") +
289                 "&message=Can't edit a read only job");
290         }
291         return job;
292     }
293
294     /**
295      * Handle job action.
296      * @param handler CrawlJobHandler to operate on.
297      * @param request Http request.
298      * @param response Http response.
299      * @param redirectBasePath Full path for where to go next if an error.
300      * E.g. "/admin/jobs/per/overview.jsp".
301      * @param currDomain Current domain. Pass null for global domain.
302      * @param reference
303      * @return The crawljob configured.
304      * @throws IOException
305      * @throws AttributeNotFoundException
306      * @throws InvocationTargetException
307      * @throws InvalidAttributeValueException
308      */
309     public static CrawlJob handleJobAction(CrawlJobHandler handler,
310             HttpServletRequest JavaDoc request, HttpServletResponse JavaDoc response,
311             String JavaDoc redirectBasePath, String JavaDoc currDomain, String JavaDoc reference)
312     throws IOException JavaDoc, AttributeNotFoundException JavaDoc, InvocationTargetException JavaDoc,
313         InvalidAttributeValueException JavaDoc {
314
315         // Load the job to manipulate
316 CrawlJob theJob =
317             checkCrawlJob(handler.getJob(request.getParameter("job")),
318                 response, redirectBasePath, currDomain);
319
320         XMLSettingsHandler settingsHandler = theJob.getSettingsHandler();
321         // If currDomain is null, then we're at top-level.
322 CrawlerSettings settings = settingsHandler
323             .getSettingsObject(currDomain);
324         
325         if(reference != null) {
326             // refinement
327 Refinement refinement = settings.getRefinement(reference);
328             settings = refinement.getSettings();
329         }
330
331         // See if we need to take any action
332 if (request.getParameter(ACTION) != null) {
333             // Need to take some action.
334 String JavaDoc action = request.getParameter(ACTION);
335             String JavaDoc subaction = request.getParameter(SUBACTION);
336             if (action.equals(FILTERS)) {
337                 // Doing something with the filters.
338 String JavaDoc map = request.getParameter(MAP);
339                 if (map != null && map.length() > 0) {
340                     String JavaDoc filter = request.getParameter(FILTER);
341                     MapType filterMap = (MapType) settingsHandler
342                         .getComplexTypeByAbsoluteName(settings, map);
343                     if (subaction.equals(ADD)) {
344                         // Add filter
345 String JavaDoc className = request.getParameter(map + ".class");
346                         String JavaDoc typeName = request.getParameter(map + ".name");
347                         if (typeName != null && typeName.length() > 0 &&
348                                 className != null && className.length() > 0) {
349                             ModuleType tmp = SettingsHandler
350                                 .instantiateModuleTypeFromClassName(
351                                     typeName, className);
352                             filterMap.addElement(settings, tmp);
353                         }
354                     } else if (subaction.equals(MOVEUP)) {
355                         // Move a filter down in a map
356 if (filter != null && filter.length() > 0) {
357                             filterMap.moveElementUp(settings, filter);
358                         }
359                     } else if (subaction.equals(MOVEDOWN)) {
360                         // Move a filter up in a map
361 if (filter != null && filter.length() > 0) {
362                             filterMap.moveElementDown(settings, filter);
363                         }
364                     } else if (subaction.equals(REMOVE)) {
365                         // Remove a filter from a map
366 if (filter != null && filter.length() > 0) {
367                             filterMap.removeElement(settings, filter);
368                         }
369                     }
370                 }
371                 // Finally save the changes to disk
372 settingsHandler.writeSettingsObject(settings);
373             } else if (action.equals(DONE)) {
374                 // Ok, done editing.
375 if(subaction.equals(CONTINUE)) {
376                     // was editting an override/refinement, simply continue
377 if (theJob.isRunning()) {
378                         handler.kickUpdate(); //Just to make sure.
379 }
380                     String JavaDoc overParam = ((currDomain != null && currDomain
381                             .length() > 0) ? "&currDomain=" + currDomain : "");
382                     String JavaDoc refParam =
383                         ((reference != null && reference.length() > 0)
384                                 ? "&reference=" + reference
385                                 : "");
386                     String JavaDoc messageParam = (refParam.length() > 0)
387                          ? "&message=Refinement changes saved"
388                          : "&message=Override changes saved";
389                     response.sendRedirect(redirectBasePath +
390                         "?job=" + theJob.getUID() +
391                          overParam +
392                          refParam +
393                          messageParam);
394                 } else {
395                     // on main, truly 'done'
396 if (theJob.isNew()) {
397                         handler.addJob(theJob);
398                         response.sendRedirect(redirectBasePath
399                                 + "?message=Job created");
400                     } else {
401                         if (theJob.isRunning()) {
402                             handler.kickUpdate();
403                         }
404                         if (theJob.isProfile()) {
405                             response.sendRedirect(redirectBasePath
406                                     + "?message=Profile modified");
407                         } else {
408                             response.sendRedirect(redirectBasePath
409                                     + "?message=Job modified");
410                         }
411                     }
412                 }
413             } else if (action.equals(GOTO)) {
414                 // Goto another page of the job/profile settings
415 String JavaDoc overParam = ((currDomain != null && currDomain
416                         .length() > 0) ? "&currDomain=" + currDomain : "");
417                 String JavaDoc refParam =
418                     ((reference != null && reference.length() > 0)
419                             ? "&reference=" + reference
420                             : "");
421                 response.sendRedirect(request.getParameter(SUBACTION) +
422                     overParam + refParam);
423             }
424         }
425         return theJob;
426     }
427     
428     /**
429      * Print complete seeds list on passed in PrintWriter.
430      * @param hndlr Current handler.
431      * @param payload What to write out.
432      * @throws AttributeNotFoundException
433      * @throws MBeanException
434      * @throws ReflectionException
435      * @throws IOException
436      * @throws IOException
437      */
438     public static void printOutSeeds(SettingsHandler hndlr, String JavaDoc payload)
439     throws AttributeNotFoundException JavaDoc, MBeanException JavaDoc, ReflectionException JavaDoc,
440     IOException JavaDoc {
441         File JavaDoc seedfile = getSeedFile(hndlr);
442         writeReader(new StringReader JavaDoc(payload),
443             new BufferedWriter JavaDoc(new FileWriter JavaDoc(seedfile)));
444     }
445     
446     /**
447      * Print complete seeds list on passed in PrintWriter.
448      * @param hndlr Current handler.
449      * @param out Writer to write out all seeds to.
450      * @throws ReflectionException
451      * @throws MBeanException
452      * @throws AttributeNotFoundException
453      * @throws IOException
454      */
455     public static void printOutSeeds(SettingsHandler hndlr, Writer JavaDoc out)
456     throws AttributeNotFoundException JavaDoc, MBeanException JavaDoc, ReflectionException JavaDoc,
457             IOException JavaDoc {
458         // getSeedStream looks for seeds on disk and on classpath.
459 InputStream JavaDoc is = getSeedStream(hndlr);
460         writeReader(new BufferedReader JavaDoc(new InputStreamReader JavaDoc(is)), out);
461     }
462     
463     /**
464      * Test whether seeds file is of a size that's reasonable
465      * to edit in an HTML textarea.
466      * @param h current settingsHandler
467      * @return true if seeds size is manageable, false otherwise
468      * @throws AttributeNotFoundException
469      * @throws MBeanException
470      * @throws ReflectionException
471      *
472      */
473     public static boolean seedsEdittableSize(SettingsHandler h)
474             throws AttributeNotFoundException JavaDoc, MBeanException JavaDoc,
475             ReflectionException JavaDoc {
476         return getSeedFile(h).length() <= (32 * 1024); // 32K
477 }
478     /**
479      * @param hndlr Settings handler.
480      * @return Seeds file.
481      * @throws ReflectionException
482      * @throws MBeanException
483      * @throws AttributeNotFoundException
484      */
485     protected static File JavaDoc getSeedFile(SettingsHandler hndlr)
486     throws AttributeNotFoundException JavaDoc, MBeanException JavaDoc, ReflectionException JavaDoc {
487         String JavaDoc seedsFileStr = (String JavaDoc)((ComplexType)hndlr.getOrder().
488             getAttribute("scope")).getAttribute("seedsfile");
489         return hndlr.getPathRelativeToWorkingDirectory(seedsFileStr);
490     }
491     
492     /**
493      * Return seeds as a stream.
494      * This method will work for case where seeds are on disk or on classpath.
495      * @param hndlr SettingsHandler. Used to find seeds.txt file.
496      * @return InputStream on current seeds file.
497      * @throws IOException
498      * @throws ReflectionException
499      * @throws MBeanException
500      * @throws AttributeNotFoundException
501      */
502     protected static InputStream JavaDoc getSeedStream(SettingsHandler hndlr)
503     throws IOException JavaDoc, AttributeNotFoundException JavaDoc, MBeanException JavaDoc,
504             ReflectionException JavaDoc {
505         InputStream JavaDoc is = null;
506         File JavaDoc seedFile = getSeedFile(hndlr);
507         if (!seedFile.exists()) {
508             // Is the file on the CLASSPATH?
509 is = SettingsHandler.class.
510                 getResourceAsStream(IoUtils.getClasspathPath(seedFile));
511         } else if(seedFile.canRead()) {
512             is = new FileInputStream JavaDoc(seedFile);
513         }
514         if (is == null) {
515             throw new IOException JavaDoc(seedFile + " does not" +
516             " exist -- neither on disk nor on CLASSPATH -- or is not" +
517             " readable.");
518         }
519         return is;
520     }
521     
522     /**
523      * Print complete seeds list on passed in PrintWriter.
524      * @param reader File to read seeds from.
525      * @param out Writer to write out all seeds to.
526      * @throws IOException
527      */
528     protected static void writeReader(Reader JavaDoc reader, Writer JavaDoc out)
529     throws IOException JavaDoc {
530         final int bufferSize = 1024 * 4;
531         char [] buffer = new char[bufferSize];
532         int read = -1;
533         while ((read = reader.read(buffer, 0, bufferSize)) != -1) {
534             out.write(buffer, 0, read);
535         }
536         out.flush();
537     }
538 }
539
Popular Tags