KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > weblech > spider > DownloadQueue


1 /*
2  * This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
3  *
4  * Copyright (c) 2001 Brian Pitcher
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24
25 // $Header: /cvsroot/weblech/weblech/src/weblech/spider/DownloadQueue.java,v 1.3 2002/06/09 11:36:23 weblech Exp $
26
27 package weblech.spider;
28
29 import java.util.*;
30 import java.net.URL JavaDoc;
31 import java.io.Serializable JavaDoc;
32
33 public class DownloadQueue implements Serializable JavaDoc
34 {
35     private SpiderConfig config;
36
37     private List interestingURLsToDownload;
38     private List averageURLsToDownload;
39     private List boringURLsToDownload;
40     private Set urlsInQueue;
41
42     public DownloadQueue(SpiderConfig config)
43     {
44         this.config = config;
45         interestingURLsToDownload = new ArrayList();
46         averageURLsToDownload = new ArrayList();
47         boringURLsToDownload = new ArrayList();
48         urlsInQueue = new HashSet();
49     }
50
51     public void queueURL(URLToDownload url)
52     {
53         URL JavaDoc u = url.getURL();
54         if(urlsInQueue.contains(u))
55         {
56             return;
57         }
58
59         if(config.isInteresting(u))
60         {
61             if(config.isDepthFirstSearch())
62             {
63                 interestingURLsToDownload.add(0, url);
64             }
65             else
66             {
67                 interestingURLsToDownload.add(url);
68             }
69         }
70         else if(config.isBoring(u))
71         {
72             if(config.isDepthFirstSearch())
73             {
74                 boringURLsToDownload.add(0, url);
75             }
76             else
77             {
78                 boringURLsToDownload.add(url);
79             }
80         }
81         else
82         {
83             if(config.isDepthFirstSearch())
84             {
85                 averageURLsToDownload.add(0, url);
86             }
87             else
88             {
89                 averageURLsToDownload.add(url);
90             }
91         }
92
93         urlsInQueue.add(u);
94     }
95
96     public void queueURLs(Collection urls)
97     {
98         for(Iterator i = urls.iterator(); i.hasNext(); )
99         {
100             URLToDownload u2d = (URLToDownload) i.next();
101             queueURL(u2d);
102         }
103     }
104
105     public URLToDownload getNextInQueue()
106     {
107         if(interestingURLsToDownload.size() > 0)
108         {
109             return returnURLFrom(interestingURLsToDownload);
110         }
111         else if(averageURLsToDownload.size() > 0)
112         {
113             return returnURLFrom(averageURLsToDownload);
114         }
115         else if(boringURLsToDownload.size() > 0)
116         {
117             return returnURLFrom(boringURLsToDownload);
118         }
119         else
120         {
121             return null;
122         }
123     }
124
125     private URLToDownload returnURLFrom(List urlList)
126     {
127         URLToDownload u2d = (URLToDownload) urlList.get(0);
128         urlList.remove(0);
129         urlsInQueue.remove(u2d.getURL());
130         return u2d;
131     }
132
133     public int size()
134     {
135         return interestingURLsToDownload.size() + averageURLsToDownload.size() + boringURLsToDownload.size();
136     }
137
138     public String JavaDoc toString()
139     {
140         return size() + " URLs";
141     }
142
143 } // End class DownloadQueue
144
Popular Tags