AbstractTracker


1   /* Copyright (C) 2003 Internet Archive.
2    *
3    * This file is part of the Heritrix web crawler (crawler.archive.org).
4    *
5    * Heritrix is free software; you can redistribute it and/or modify
6    * it under the terms of the GNU Lesser Public License as published by
7    * the Free Software Foundation; either version 2.1 of the License, or
8    * any later version.
9    *
10   * Heritrix is distributed in the hope that it will be useful,
11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13   * GNU Lesser Public License for more details.
14   *
15   * You should have received a copy of the GNU Lesser Public License
16   * along with Heritrix; if not, write to the Free Software
17   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18   */
19  package org.archive.crawler.framework;
20  
21  import java.io.Serializable  ;
22  import java.util.Date  ;
23  import java.util.EventObject  ;
24  import java.util.logging.Level  ;
25  
26  import javax.management.AttributeNotFoundException  ;
27  
28  import org.archive.crawler.event.CrawlStatusListener;
29  import org.archive.crawler.framework.exceptions.FatalConfigurationException;
30  import org.archive.crawler.settings.ModuleType;
31  import org.archive.crawler.settings.SimpleType;
32  import org.archive.crawler.settings.Type;
33  import org.archive.util.ArchiveUtils;
34  import org.archive.util.PaddingStringBuffer;
35  import org.xbill.DNS.DClass;
36  import org.xbill.DNS.Lookup;
37  
38  /**
39   * A partial implementation of the StatisticsTracking interface.
40   * <p>
41   * It covers the thread handling. (Launching, pausing etc.)  Included in this is
42   * keeping track of the total time spent (actually) crawling.  Several methods
43   * to access the time started, finished etc. are provided.
44   * <p>
45   * To handle the thread work the class implements the CrawlStatusListener and
46   * uses it's events to pause, resume and stop logging of statistics. The run()
47   * method will call logActivity() at intervals specified in the crawl order.
48   * <p>
49   * Implementation of logActivity (the actual logging) as well as listening for
50   * CrawlURIDisposition events is not addressed.
51   *
52   * @author Kristinn Sigurdsson
53   *
54   * @see org.archive.crawler.framework.StatisticsTracking
55   * @see org.archive.crawler.admin.StatisticsTracker
56   */
57  public abstract class AbstractTracker extends ModuleType
58  implements StatisticsTracking, CrawlStatusListener, Serializable   {
59      /** Default period between logging stat values */
60      public static final Integer   DEFAULT_STATISTICS_REPORT_INTERVAL =
61          new Integer  (20);
62      /** Attribute name for logging interval in seconds setting
63       */
64      public static final String   ATTR_STATS_INTERVAL = "interval-seconds";
65  
66      /** A reference to the CrawlContoller of the crawl that we are to track
67       * statistics for.
68       */
69      protected transient CrawlController controller;
70  
71      // Keep track of time.
72      protected long crawlerStartTime;
73      protected long crawlerEndTime = -1; // Until crawl ends, this value is -1.
74      protected long crawlerPauseStarted = 0;
75      protected long crawlerTotalPausedTime = 0;
76  
77      /** Timestamp of when this logger last wrote something to the log */
78      protected long lastLogPointTime;
79  
80      protected boolean shouldrun = true;
81  
82      /**
83       * @param name
84       * @param description
85       */
86      public AbstractTracker(String   name, String   description) {
87          super(name, description);
88          Type e = addElementToDefinition(new SimpleType(ATTR_STATS_INTERVAL,
89                  "The interval between writing progress information to log.",
90                  DEFAULT_STATISTICS_REPORT_INTERVAL));
91          e.setOverrideable(false);
92      }
93  
94      /**
95       * Sets up the Logger (including logInterval) and registers with the
96       * CrawlController for CrawlStatus and CrawlURIDisposition events.
97       *
98       * @param c A crawl controller instance.
99       * @throws FatalConfigurationException Not thrown here. For overrides that
100      * go to settings system for configuration.
101      * @see CrawlStatusListener
102      * @see org.archive.crawler.event.CrawlURIDispositionListener
103      */
104     public void initialize(CrawlController c)
105     throws FatalConfigurationException {
106         this.controller = c;
107 
108         // Add listeners
109         this.controller.addCrawlStatusListener(this);
110     }
111     
112     /**
113      * Start thread.  Will call logActivity() at intervals specified by
114      * logInterval
115      *
116      */
117     public void run() {
118         // Don't start logging if we have no logger
119         if (this.controller == null) {
120             return;
121         }
122 
123         shouldrun = true; //If we are starting, this should always be true.
124 
125         // Log the legend
126         this.controller.logProgressStatistics(progressStatisticsLegend());
127         lastLogPointTime = System.currentTimeMillis(); // The first interval begins now.
128 
129         // Keep logging until someone calls stop()
130         while (shouldrun) {
131             // Pause before writing the first entry (so we have real numbers)
132             // and then pause between entries
133             try {
134                 Thread.sleep(getLogWriteInterval() * 1000);
135             } catch (InterruptedException   e) {
136                 e.printStackTrace();
137                 controller.runtimeErrors.log(Level.INFO,
138                     "Periodic stat logger interrupted while sleeping.");
139             }
140 
141             // In case stop() was invoked while the thread was sleeping or we
142             // are paused.
143             if (shouldrun && getCrawlPauseStartedTime() == 0) {
144                 progressStatisticsEvent(new EventObject  (this));
145             }
146         }
147     }
148 
149     /**
150      * @return legend for progress-statistics lines/log
151      */
152     public String   progressStatisticsLegend() {
153         return "           timestamp" +
154             "  discovered   " +
155             "   queued   downloaded       doc/s(avg)  KB/s(avg) " +
156             "  dl-failures   busy-thread   mem-use-KB  heap-size-KB " +
157             "  congestion   max-depth   avg-depth";
158     }
159 
160     /**
161      * Notify tracker that crawl has begun. Must be called
162      * outside tracker's own thread, to ensure it is noted
163      * before other threads start interacting with tracker. 
164      */
165     public void noteStart() {
166         if (this.crawlerStartTime == 0) {
167             // Note the time the crawl starts (only if not already set)
168             this.crawlerStartTime = System.currentTimeMillis();
169         }
170     }
171 
172     /**
173      * A method for logging current crawler state.
174      *
175      * This method will be called by run() at intervals specified in
176      * the crawl order file.  It is also invoked when pausing or
177      * stopping a crawl to capture the state at that point.  Default behavior is
178      * call to {@link CrawlController#logProgressStatistics} so CrawlController
179      * can act on progress statistics event.
180      * <p>
181      * It is recommended that for implementations of this method it be
182      * carefully considered if it should be synchronized in whole or in
183      * part
184      * @param e Progress statistics event.
185      */
186     protected synchronized void progressStatisticsEvent(final EventObject   e) {
187         this.controller.progressStatisticsEvent(e);
188         // temporary workaround for 
189         // [ 996161 ] Fix DNSJava issues (memory) -- replace with JNDI-DNS?
190         // http://sourceforge.net/support/tracker.php?aid=996161
191         Lookup.getDefaultCache(DClass.IN).clearCache();
192     }
193 
194     /**
195      * Get the starting time of the crawl (as given by
196      * <code>System.currentTimeMillis()</code> when the crawl started).
197      * @return time fo the crawl's start
198      */
199     public long getCrawlStartTime() {
200         return this.crawlerStartTime;
201     }
202 
203     /**
204      * If crawl has ended it will return the time it ended (given by
205      * <code>System.currentTimeMillis()</code> at that time).
206      * <br>
207      * If crawl is still going on it will return the same as
208      * <code>System.currentTimeMillis()</code> at the time of the call.
209      * @return The time of the crawl ending or the current time if the crawl has
210      *         not ended.
211      */
212     public long getCrawlEndTime() {
213         return (this.crawlerEndTime == -1)?
214             System.currentTimeMillis(): this.crawlerEndTime;
215     }
216 
217     /**
218      * Returns the number of milliseconds that the crawl spent paused or
219      * otherwise in a nonactive state.
220      * @return the number of msec. that the crawl was paused or otherwise
221      *         suspended.
222      */
223     public long getCrawlTotalPauseTime() {
224         return this.crawlerTotalPausedTime;
225     }
226 
227     /**
228      * Get the time when the the crawl was last paused/suspended (as given by
229      * <code>System.currentTimeMillis()</code> at that time). Will be 0 if the
230      * crawl is not currently paused.
231      * @return time of the crawl's last pause/suspend or 0 if the crawl is not
232      *         currently paused.
233      */
234     public long getCrawlPauseStartedTime() {
235         return this.crawlerPauseStarted;
236     }
237 
238     public long getCrawlerTotalElapsedTime() {
239         if (getCrawlStartTime() == 0) {
240             // if no start time set yet, consider elapsed time zero
241             return 0;
242         }
243         
244         return (getCrawlPauseStartedTime() != 0)?
245             // Are currently paused, calculate time up to last pause
246             (getCrawlPauseStartedTime() - getCrawlTotalPauseTime() -
247                 getCrawlStartTime()):
248             // Not paused, calculate total time.
249             (getCrawlEndTime() - getCrawlTotalPauseTime() - getCrawlStartTime());
250     }
251 
252     /**
253      * The number of seconds to wait between writing snapshot data to log file.
254      * @return the number of seconds to wait between writing snapshot data to
255      * log file.
256      */
257     protected int getLogWriteInterval() {
258         int logInterval;
259         try {
260             logInterval =
261                 ((Integer  ) getAttribute(null, ATTR_STATS_INTERVAL)).intValue();
262         } catch (AttributeNotFoundException   e) {
263             logInterval = 10;
264         }
265         return logInterval;
266     }
267 
268     /**
269      * @see org.archive.crawler.event.CrawlStatusListener#crawlPausing(java.lang.String)
270      */
271     public void crawlPausing(String   statusMessage) {
272         logNote("CRAWL WAITING - " + statusMessage);
273     }
274 
275     protected void logNote(final String   note) {
276         this.controller.logProgressStatistics(new PaddingStringBuffer()
277                      .append(ArchiveUtils.TIMESTAMP14.format(new Date  ()))
278                      .append(" ")
279                      .append(note)
280                      .toString());
281     }
282 
283     public void crawlPaused(String   statusMessage) {
284         crawlerPauseStarted = System.currentTimeMillis();
285         progressStatisticsEvent(new EventObject  (this));
286         logNote("CRAWL PAUSED - " + statusMessage);
287     }
288 
289     public void crawlResuming(String   statusMessage) {
290         tallyCurrentPause();
291         logNote("CRAWL RESUMED - " + statusMessage);
292         lastLogPointTime = System.currentTimeMillis();
293     }
294 
295     /**
296      * For a current pause (if any), add paused time to total and reset
297      */
298     protected void tallyCurrentPause() {
299         if (this.crawlerPauseStarted > 0) {
300             // Ok, we managed to actually pause before resuming.
301             this.crawlerTotalPausedTime
302                 += (System.currentTimeMillis() - this.crawlerPauseStarted);
303         }
304         this.crawlerPauseStarted = 0;
305     }
306 
307     public void crawlEnding(String   sExitMessage) {
308         logNote("CRAWL ENDING - " + sExitMessage);
309     }
310 
311     /**
312      * @see org.archive.crawler.event.CrawlStatusListener#crawlEnded(java.lang.String)
313      */
314     public void crawlEnded(String   sExitMessage) {
315         // Note the time when the crawl stops.
316         crawlerEndTime = System.currentTimeMillis();
317         progressStatisticsEvent(new EventObject  (this));
318         logNote("CRAWL ENDED - " + sExitMessage);
319         shouldrun = false;
320         dumpReports();
321         finalCleanup();
322     }
323 
324     public void crawlStarted(String   message) {
325         tallyCurrentPause();
326         noteStart();
327     }
328     
329     /**
330      * Dump reports, if any, on request or at crawl end. 
331      */
332     protected void dumpReports() {
333         // by default do nothing; subclasses may override
334     }
335 
336     /**
337      * Cleanup resources used, at crawl end. 
338      */
339     protected void finalCleanup() {
340         controller = null; // Facilitate GC.
341     }
342 
343     /**
344      * @see org.archive.crawler.framework.StatisticsTracking#crawlDuration()
345      */
346     public long crawlDuration() {
347         return getCrawlerTotalElapsedTime();
348     }
349 }
350
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags