KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > archive > crawler > framework > ToePool


1 /* Copyright (C) 2003 Internet Archive.
2  *
3  * This file is part of the Heritrix web crawler (crawler.archive.org).
4  *
5  * Heritrix is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU Lesser Public License as published by
7  * the Free Software Foundation; either version 2.1 of the License, or
8  * any later version.
9  *
10  * Heritrix is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  * GNU Lesser Public License for more details.
14  *
15  * You should have received a copy of the GNU Lesser Public License
16  * along with Heritrix; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18  *
19  * ToePool.java
20  * Created on Oct 1, 2003
21  *
22  * $Header: /cvsroot/archive-crawler/ArchiveOpenCrawler/src/java/org/archive/crawler/framework/ToePool.java,v 1.31 2005/11/10 01:10:02 stack-sf Exp $
23  */

24 package org.archive.crawler.framework;
25
26 import java.io.PrintWriter JavaDoc;
27 import java.util.Date JavaDoc;
28 import java.util.Iterator JavaDoc;
29 import java.util.TreeSet JavaDoc;
30
31 import org.archive.util.ArchiveUtils;
32 import org.archive.util.Histotable;
33 import org.archive.util.Reporter;
34
35 /**
36  * A collection of ToeThreads. The class manages the ToeThreads currently
37  * running. Including increasing and decreasing their number, keeping track
38  * of their state and it can be used to kill hung threads.
39  *
40  * @author Gordon Mohr
41  * @author Kristinn Sigurdsson
42  *
43  * @see org.archive.crawler.framework.ToeThread
44  */

45 public class ToePool extends ThreadGroup JavaDoc implements Reporter {
46     /** run worker thread slightly lower than usual */
47     public static int DEFAULT_TOE_PRIORITY = Thread.NORM_PRIORITY - 1;
48     
49     protected CrawlController controller;
50     protected int nextSerialNumber = 1;
51     protected int targetSize = 0;
52
53     /**
54      * Constructor. Creates a pool of ToeThreads.
55      *
56      * @param c A reference to the CrawlController for the current crawl.
57      */

58     public ToePool(CrawlController c) {
59         super("ToeThreads");
60         this.controller = c;
61     }
62     
63     public void cleanup() {
64         this.controller = null;
65     }
66
67     /**
68      * @return The number of ToeThreads that are not available (Approximation).
69      */

70     public int getActiveToeCount() {
71         Thread JavaDoc[] toes = getToes();
72         int count = 0;
73         for (int i = 0; i < toes.length; i++) {
74             if((toes[i] instanceof ToeThread) &&
75                     ((ToeThread)toes[i]).isActive()) {
76                 count++;
77             }
78         }
79         return count;
80     }
81
82     /**
83      * @return The number of ToeThreads. This may include killed ToeThreads
84      * that were not replaced.
85      */

86     public int getToeCount() {
87         Thread JavaDoc[] toes = getToes();
88         int count = 0;
89         for (int i = 0; i<toes.length; i++) {
90             if((toes[i] instanceof ToeThread)) {
91                 count++;
92             }
93         }
94         return count;
95     }
96     
97     private Thread JavaDoc[] getToes() {
98         Thread JavaDoc[] toes = new Thread JavaDoc[activeCount()+10];
99         this.enumerate(toes);
100         return toes;
101     }
102
103     /**
104      * Change the number of ToeThreads.
105      *
106      * @param newsize The new number of ToeThreads.
107      */

108     public void setSize(int newsize)
109     {
110         targetSize = newsize;
111         int difference = newsize - getToeCount();
112         if (difference > 0) {
113             // must create threads
114
for(int i = 1; i <= difference; i++) {
115                 startNewThread();
116             }
117         } else {
118             // must retire extra threads
119
int retainedToes = targetSize;
120             Thread JavaDoc[] toes = this.getToes();
121             for (int i = 0; i < toes.length ; i++) {
122                 if(!(toes[i] instanceof ToeThread)) {
123                     continue;
124                 }
125                 retainedToes--;
126                 if (retainedToes>=0) {
127                     continue; // this toe is spared
128
}
129                 // otherwise:
130
ToeThread tt = (ToeThread)toes[i];
131                 tt.retire();
132             }
133         }
134     }
135
136     /**
137      * Kills specified thread. Killed thread can be optionally replaced with a
138      * new thread.
139      *
140      * <p><b>WARNING:</b> This operation should be used with great care. It may
141      * destabilize the crawler.
142      *
143      * @param threadNumber Thread to kill
144      * @param replace If true then a new thread will be created to take the
145      * killed threads place. Otherwise the total number of threads
146      * will decrease by one.
147      */

148     public void killThread(int threadNumber, boolean replace){
149
150         Thread JavaDoc[] toes = getToes();
151         for (int i = 0; i< toes.length; i++) {
152             if(! (toes[i] instanceof ToeThread)) {
153                 continue;
154             }
155             ToeThread toe = (ToeThread) toes[i];
156             if(toe.getSerialNumber()==threadNumber) {
157                 toe.kill();
158             }
159         }
160
161         if(replace){
162             // Create a new toe thread to take its place. Replace toe
163
startNewThread();
164         }
165     }
166
167     private synchronized void startNewThread() {
168         ToeThread newThread = new ToeThread(this, nextSerialNumber++);
169         newThread.setPriority(DEFAULT_TOE_PRIORITY);
170         newThread.start();
171     }
172
173     /**
174      * @return Instance of CrawlController.
175      */

176     public CrawlController getController() {
177         return controller;
178     }
179     
180     //
181
// Reporter implementation
182
//
183

184     public static String JavaDoc STANDARD_REPORT = "standard";
185     public static String JavaDoc COMPACT_REPORT = "compact";
186     protected static String JavaDoc[] REPORTS = {STANDARD_REPORT,COMPACT_REPORT};
187
188     public String JavaDoc[] getReports() {
189         return REPORTS;
190     }
191
192     public void reportTo(String JavaDoc name, PrintWriter JavaDoc writer) {
193         if(COMPACT_REPORT.equals(name)) {
194             compactReportTo(writer);
195             return;
196         }
197         if(name!=null && !STANDARD_REPORT.equals(name)) {
198             writer.print(name);
199             writer.print(" not recognized: giving standard report/n");
200         }
201         standardReportTo(writer);
202     }
203             
204     /* (non-Javadoc)
205      * @see org.archive.util.Reporter#reportTo(java.io.Writer)
206      */

207     protected void standardReportTo(PrintWriter JavaDoc writer) {
208         writer.print("Toe threads report - "
209                 + ArchiveUtils.TIMESTAMP12.format(new Date JavaDoc()) + "\n");
210         writer.print(" Job being crawled: "
211                 + this.controller.getOrder().getCrawlOrderName() + "\n");
212         writer.print(" Number of toe threads in pool: " + getToeCount() + " ("
213                 + getActiveToeCount() + " active)\n");
214
215         Thread JavaDoc[] toes = this.getToes();
216         synchronized (toes) {
217             for (int i = 0; i < toes.length; i++) {
218                 if (!(toes[i] instanceof ToeThread)) {
219                     continue;
220                 }
221                 ToeThread tt = (ToeThread) toes[i];
222                 if (tt != null) {
223                     writer
224                             .print(" ToeThread #" + tt.getSerialNumber()
225                                     + "\n");
226                     tt.reportTo(writer);
227                 }
228             }
229         }
230     }
231     
232     /* (non-Javadoc)
233      * @see org.archive.util.Reporter#reportTo(java.io.Writer)
234      */

235     protected void compactReportTo(PrintWriter JavaDoc writer) {
236         writer.print(getToeCount() + " threads (" + getActiveToeCount()
237                 + " active)\n");
238
239         Thread JavaDoc[] toes = this.getToes();
240         boolean legendWritten = false;
241         // TODO: sort by activity: those with curi the longest at front
242
synchronized (toes) {
243             for (int i = 0; i < toes.length; i++) {
244                 if (!(toes[i] instanceof ToeThread)) {
245                     continue;
246                 }
247                 ToeThread tt = (ToeThread) toes[i];
248                 if (tt != null) {
249                     if(!legendWritten) {
250                         writer.println(tt.singleLineLegend());
251                         legendWritten = true;
252                     }
253                     tt.singleLineReportTo(writer);
254                 }
255             }
256         }
257     }
258
259     public void singleLineReportTo(PrintWriter JavaDoc w) {
260         Histotable ht = new Histotable();
261         Thread JavaDoc[] toes = getToes();
262         for (int i = 0; i < toes.length; i++) {
263
264             if(!(toes[i] instanceof ToeThread)) {
265                 continue;
266             }
267             ToeThread tt = (ToeThread)toes[i];
268             if(tt!=null) {
269                 ht.tally(tt.getStep());
270             }
271         }
272         TreeSet JavaDoc sorted = ht.getSorted();
273         w.print(getToeCount());
274         w.print(" threads: ");
275         w.print(Histotable.entryString(sorted.first()));
276         if(sorted.size()>1) {
277             Iterator JavaDoc iter = sorted.iterator();
278             iter.next();
279             w.print("; ");
280             w.print(Histotable.entryString(iter.next()));
281         }
282         if(sorted.size()>2) {
283             w.print("; etc...");
284         }
285     }
286
287     /* (non-Javadoc)
288      * @see org.archive.util.Reporter#singleLineLegend()
289      */

290     public String JavaDoc singleLineLegend() {
291         return "total: mostCommonStateTotal secondMostCommonStateTotal";
292     }
293     
294     public String JavaDoc singleLineReport() {
295         return ArchiveUtils.singleLineReport(this);
296     }
297
298     public void reportTo(PrintWriter JavaDoc writer) {
299         reportTo(null,writer);
300     }
301 }
302
Popular Tags