KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > net > javacoding > jspider > core > task > impl > SchedulerImpl


1 package net.javacoding.jspider.core.task.impl;
2
3 import net.javacoding.jspider.core.exception.*;
4 import net.javacoding.jspider.core.task.*;
5 import net.javacoding.jspider.core.task.work.DecideOnSpideringTask;
6
7 import java.net.URL JavaDoc;
8 import java.util.*;
9
10 /**
11  * Default implementation of a Task scheduler
12  *
13  * $Id: SchedulerImpl.java,v 1.17 2003/04/25 21:29:04 vanrogu Exp $
14  *
15  * @author Günther Van Roey
16  */
17 public class SchedulerImpl implements Scheduler {
18
19     /** List of fetch tasks to be carried out. */
20     protected List fetchTasks;
21
22     /** List of thinker tasks to be carried out. */
23     protected List thinkerTasks;
24
25     /** Set of tasks that have been assigned. */
26     protected Set assignedSpiderTasks;
27     protected Set assignedThinkerTasks;
28
29     protected int spiderTasksDone;
30     protected int thinkerTasksDone;
31
32     protected Map blocked;
33
34     int blockedCount = 0;
35
36     public int getBlockedCount( ) {
37         return blockedCount;
38     }
39
40     public int getAssignedCount( ) {
41         return assignedSpiderTasks.size() + assignedThinkerTasks.size();
42     }
43
44     public int getJobCount() {
45         return getThinkerJobCount() + getSpiderJobCount();
46     }
47
48     public int getThinkerJobCount() {
49         return thinkerTasksDone + assignedThinkerTasks.size ( ) + thinkerTasks.size();
50     }
51
52     public int getSpiderJobCount() {
53         return spiderTasksDone + assignedSpiderTasks.size ( ) + fetchTasks.size();
54     }
55
56     public int getJobsDone() {
57       return getSpiderJobsDone() + getThinkerJobsDone();
58     }
59
60     public int getSpiderJobsDone() {
61         return spiderTasksDone;
62     }
63
64     public int getThinkerJobsDone() {
65         return thinkerTasksDone;
66     }
67
68
69     /**
70      * Public constructor?
71      */
72     public SchedulerImpl() {
73         fetchTasks = new ArrayList();
74         thinkerTasks = new ArrayList();
75         assignedThinkerTasks = new HashSet();
76         assignedSpiderTasks = new HashSet();
77         blocked = new HashMap();
78     }
79
80     public void block(URL JavaDoc siteURL, DecideOnSpideringTask task) {
81         ArrayList al = (ArrayList)blocked.get(siteURL);
82         if ( al == null ) {
83             al = new ArrayList();
84             blocked.put(siteURL, al);
85         }
86         int before = al.size();
87         al.add(task);
88         int after = al.size();
89
90         int diff = after-before;
91         blockedCount+=diff;
92     }
93
94     public DecideOnSpideringTask[] unblock(URL JavaDoc siteURL) {
95         ArrayList al = (ArrayList)blocked.remove(siteURL);
96         if ( al == null ) {
97             return new DecideOnSpideringTask[0];
98         } else {
99           blockedCount-=al.size();
100           return (DecideOnSpideringTask[]) al.toArray(new DecideOnSpideringTask[al.size()]);
101         }
102     }
103
104     /**
105      * Schedules a task to be processed.
106      * @param task task to be scheduled
107      */
108     public synchronized void schedule(WorkerTask task) {
109         if (task.getType() == WorkerTask.WORKERTASK_SPIDERTASK ) {
110             fetchTasks.add(task);
111         } else {
112             thinkerTasks.add(task);
113         }
114     }
115
116     /**
117      * Flags a task as done.
118      * @param task task that was complete
119      */
120     public synchronized void flagDone(WorkerTask task) {
121         if (task.getType() == WorkerTask.WORKERTASK_THINKERTASK ) {
122             assignedThinkerTasks.remove(task);
123             thinkerTasksDone++;
124         }else{
125             assignedSpiderTasks.remove(task);
126             spiderTasksDone++;
127         }
128     }
129
130     public synchronized WorkerTask getThinkerTask() throws TaskAssignmentException {
131         if (thinkerTasks.size() > 0) {
132             WorkerTask task = (WorkerTask) thinkerTasks.remove(0);
133             assignedThinkerTasks.add(task);
134             return task;
135         }
136         if (allTasksDone()) {
137             throw new SpideringDoneException();
138         } else {
139             throw new NoSuitableItemFoundException();
140         }
141     }
142
143     /**
144      * Returns a fetch task to be carried out.
145      * @return WorkerTask task to be done
146      * @throws TaskAssignmentException notifies when the work is done or there
147      * are no current outstanding tasks.
148      */
149     public synchronized WorkerTask getFethTask() throws TaskAssignmentException {
150         if (fetchTasks.size() > 0) {
151             WorkerTask task = (WorkerTask) fetchTasks.remove(0);
152             assignedSpiderTasks.add(task);
153             return task;
154         }
155         if (allTasksDone()) {
156             throw new SpideringDoneException();
157         } else {
158             throw new NoSuitableItemFoundException();
159         }
160     }
161
162
163     /**
164      * Determines whether all the tasks are done. If there are no more tasks
165      * scheduled for process, and no ongoing tasks, it is impossible that new
166      * work will arrive, so the spidering is done.
167      * @return boolean value determining whether all work is done
168      */
169     public synchronized boolean allTasksDone() {
170         return (fetchTasks.size() == 0 &&
171                 thinkerTasks.size() == 0 &&
172                 assignedSpiderTasks.size() == 0 &&
173                 assignedThinkerTasks.size() == 0 &&
174                 blocked.size() == 0 );
175     }
176
177     /*
178     public synchronized String toString ( ) {
179         StringBuffer sb = new StringBuffer();
180         Iterator it = this.thinkerTasks.iterator();
181         while ( it.hasNext() ) {
182             System.out.println("TH . " + it.next().getClass());
183         }
184         it = this.assignedThinkerTasks.iterator();
185         while ( it.hasNext() ) {
186             System.out.println("TH A " + it.next().getClass());
187         }
188         it = this.fetchTasks.iterator();
189         while ( it.hasNext() ) {
190             System.out.println("SP . " + it.next().getClass());
191         }
192         it = this.assignedSpiderTasks.iterator();
193         while ( it.hasNext() ) {
194             System.out.println("SP A " + it.next().getClass());
195         }
196         return sb.toString();
197     } */
198 }
199
Popular Tags