1 24 package org.archive.crawler.framework; 25 26 import java.io.PrintWriter ; 27 import java.util.Date ; 28 import java.util.Iterator ; 29 import java.util.TreeSet ; 30 31 import org.archive.util.ArchiveUtils; 32 import org.archive.util.Histotable; 33 import org.archive.util.Reporter; 34 35 45 public class ToePool extends ThreadGroup implements Reporter { 46 47 public static int DEFAULT_TOE_PRIORITY = Thread.NORM_PRIORITY - 1; 48 49 protected CrawlController controller; 50 protected int nextSerialNumber = 1; 51 protected int targetSize = 0; 52 53 58 public ToePool(CrawlController c) { 59 super("ToeThreads"); 60 this.controller = c; 61 } 62 63 public void cleanup() { 64 this.controller = null; 65 } 66 67 70 public int getActiveToeCount() { 71 Thread [] toes = getToes(); 72 int count = 0; 73 for (int i = 0; i < toes.length; i++) { 74 if((toes[i] instanceof ToeThread) && 75 ((ToeThread)toes[i]).isActive()) { 76 count++; 77 } 78 } 79 return count; 80 } 81 82 86 public int getToeCount() { 87 Thread [] toes = getToes(); 88 int count = 0; 89 for (int i = 0; i<toes.length; i++) { 90 if((toes[i] instanceof ToeThread)) { 91 count++; 92 } 93 } 94 return count; 95 } 96 97 private Thread [] getToes() { 98 Thread [] toes = new Thread [activeCount()+10]; 99 this.enumerate(toes); 100 return toes; 101 } 102 103 108 public void setSize(int newsize) 109 { 110 targetSize = newsize; 111 int difference = newsize - getToeCount(); 112 if (difference > 0) { 113 for(int i = 1; i <= difference; i++) { 115 startNewThread(); 116 } 117 } else { 118 int retainedToes = targetSize; 120 Thread [] toes = this.getToes(); 121 for (int i = 0; i < toes.length ; i++) { 122 if(!(toes[i] instanceof ToeThread)) { 123 continue; 124 } 125 retainedToes--; 126 if (retainedToes>=0) { 127 continue; } 129 ToeThread tt = (ToeThread)toes[i]; 131 tt.retire(); 132 } 133 } 134 } 135 136 148 public void killThread(int threadNumber, boolean replace){ 149 150 Thread [] toes = getToes(); 151 for (int i = 0; i< toes.length; i++) { 152 if(! (toes[i] instanceof ToeThread)) { 153 continue; 154 } 155 ToeThread toe = (ToeThread) toes[i]; 156 if(toe.getSerialNumber()==threadNumber) { 157 toe.kill(); 158 } 159 } 160 161 if(replace){ 162 startNewThread(); 164 } 165 } 166 167 private synchronized void startNewThread() { 168 ToeThread newThread = new ToeThread(this, nextSerialNumber++); 169 newThread.setPriority(DEFAULT_TOE_PRIORITY); 170 newThread.start(); 171 } 172 173 176 public CrawlController getController() { 177 return controller; 178 } 179 180 184 public static String STANDARD_REPORT = "standard"; 185 public static String COMPACT_REPORT = "compact"; 186 protected static String [] REPORTS = {STANDARD_REPORT,COMPACT_REPORT}; 187 188 public String [] getReports() { 189 return REPORTS; 190 } 191 192 public void reportTo(String name, PrintWriter writer) { 193 if(COMPACT_REPORT.equals(name)) { 194 compactReportTo(writer); 195 return; 196 } 197 if(name!=null && !STANDARD_REPORT.equals(name)) { 198 writer.print(name); 199 writer.print(" not recognized: giving standard report/n"); 200 } 201 standardReportTo(writer); 202 } 203 204 207 protected void standardReportTo(PrintWriter writer) { 208 writer.print("Toe threads report - " 209 + ArchiveUtils.TIMESTAMP12.format(new Date ()) + "\n"); 210 writer.print(" Job being crawled: " 211 + this.controller.getOrder().getCrawlOrderName() + "\n"); 212 writer.print(" Number of toe threads in pool: " + getToeCount() + " (" 213 + getActiveToeCount() + " active)\n"); 214 215 Thread [] toes = this.getToes(); 216 synchronized (toes) { 217 for (int i = 0; i < toes.length; i++) { 218 if (!(toes[i] instanceof ToeThread)) { 219 continue; 220 } 221 ToeThread tt = (ToeThread) toes[i]; 222 if (tt != null) { 223 writer 224 .print(" ToeThread #" + tt.getSerialNumber() 225 + "\n"); 226 tt.reportTo(writer); 227 } 228 } 229 } 230 } 231 232 235 protected void compactReportTo(PrintWriter writer) { 236 writer.print(getToeCount() + " threads (" + getActiveToeCount() 237 + " active)\n"); 238 239 Thread [] toes = this.getToes(); 240 boolean legendWritten = false; 241 synchronized (toes) { 243 for (int i = 0; i < toes.length; i++) { 244 if (!(toes[i] instanceof ToeThread)) { 245 continue; 246 } 247 ToeThread tt = (ToeThread) toes[i]; 248 if (tt != null) { 249 if(!legendWritten) { 250 writer.println(tt.singleLineLegend()); 251 legendWritten = true; 252 } 253 tt.singleLineReportTo(writer); 254 } 255 } 256 } 257 } 258 259 public void singleLineReportTo(PrintWriter w) { 260 Histotable ht = new Histotable(); 261 Thread [] toes = getToes(); 262 for (int i = 0; i < toes.length; i++) { 263 264 if(!(toes[i] instanceof ToeThread)) { 265 continue; 266 } 267 ToeThread tt = (ToeThread)toes[i]; 268 if(tt!=null) { 269 ht.tally(tt.getStep()); 270 } 271 } 272 TreeSet sorted = ht.getSorted(); 273 w.print(getToeCount()); 274 w.print(" threads: "); 275 w.print(Histotable.entryString(sorted.first())); 276 if(sorted.size()>1) { 277 Iterator iter = sorted.iterator(); 278 iter.next(); 279 w.print("; "); 280 w.print(Histotable.entryString(iter.next())); 281 } 282 if(sorted.size()>2) { 283 w.print("; etc..."); 284 } 285 } 286 287 290 public String singleLineLegend() { 291 return "total: mostCommonStateTotal secondMostCommonStateTotal"; 292 } 293 294 public String singleLineReport() { 295 return ArchiveUtils.singleLineReport(this); 296 } 297 298 public void reportTo(PrintWriter writer) { 299 reportTo(null,writer); 300 } 301 } 302 | Popular Tags |