|
1 package org.roller.business;
2
3 import java.net.URL  ;
4 import java.util.Calendar ;
5 import java.util.Date ;
6 import java.util.HashMap ;
7 import java.util.Iterator ;
8 import java.util.List ;
9 import java.util.Map ;
10 import java.util.Set ;
11 import java.util.TreeSet ;
12 import java.text.MessageFormat ;
13
14 import org.apache.commons.logging.Log;
15 import org.apache.commons.logging.LogFactory;
16 import org.roller.RollerException;
17 import org.roller.model.PlanetManager;
18 import org.roller.model.Roller;
19 import org.roller.pojos.PlanetConfigData;
20 import org.roller.pojos.PlanetEntryData;
21 import org.roller.pojos.PlanetGroupData;
22 import org.roller.pojos.PlanetSubscriptionData;
23 import org.roller.util.rome.DiskFeedInfoCache;
24 import org.roller.util.LRUCache2;
25
26 import com.sun.syndication.feed.synd.SyndEntry;
27 import com.sun.syndication.feed.synd.SyndFeed;
28 import com.sun.syndication.fetcher.FeedFetcher;
29 import com.sun.syndication.fetcher.impl.FeedFetcherCache;
30 import com.sun.syndication.fetcher.impl.HttpURLFeedFetcher;
31 import com.sun.syndication.fetcher.impl.SyndFeedInfo;
32
33
37 public abstract class PlanetManagerImpl implements PlanetManager
38 {
39 protected Roller roller = null;
40 protected PersistenceStrategy strategy;
41 protected Date lastUpdated = new Date ();
42 protected Map lastUpdatedByGroup = new HashMap ();
43
44 protected LRUCache2 aggregationsByGroup =
47 new LRUCache2(20, 30 * 60 * 1000);
48
49 protected LRUCache2 topSubscriptionsByGroup =
52 new LRUCache2(20, 30 * 60 * 1000);
53
54 private static Log logger =
55 LogFactory.getFactory().getInstance(PlanetManagerImpl.class);
56
57 public PlanetManagerImpl()
58 {
59 }
60
61 public PlanetManagerImpl(PersistenceStrategy strategy, Roller roller)
62 {
63 this.strategy = strategy;
64 this.roller = roller;
65 }
66
67 public void refreshEntries() throws RollerException
68 {
69 Date now = new Date ();
70 long startTime = System.currentTimeMillis();
71 PlanetConfigData config = getConfiguration();
72 if (config == null || config.getCacheDir() == null)
73 {
74 logger.warn("Planet cache directory not set, aborting refresh");
75 return;
76 }
77 FeedFetcherCache feedInfoCache =
78 new DiskFeedInfoCache(config.getCacheDir());
79
80 if (config.getProxyHost()!=null && config.getProxyPort() > 0)
81 {
82 System.setProperty("proxySet", "true");
83 System.setProperty("http.proxyHost", config.getProxyHost());
84 System.setProperty("http.proxyPort",
85 Integer.toString(config.getProxyPort()));
86 }
87
88 System.setProperty("sun.net.client.defaultConnectTimeout", "15000");
89 System.setProperty("sun.net.client.defaultReadTimeout", "15000");
90
91 FeedFetcher feedFetcher = new HttpURLFeedFetcher(feedInfoCache);
92 feedFetcher.setUsingDeltaEncoding(false);
94 feedFetcher.setUserAgent("Roller Planet 1.1-dev");
95
96 Iterator subs = getAllSubscriptions();
98 while (subs.hasNext())
99 {
100 long subStartTime = System.currentTimeMillis();
101
102 Set newEntries = new TreeSet ();
104 PlanetSubscriptionData sub = (PlanetSubscriptionData)subs.next();
105 SyndFeed feed = null;
106 URL feedUrl = null;
107 Date lastUpdated = now;
108 try
109 {
110 feedUrl = new URL (sub.getFeedUrl());
111 logger.debug("Get feed from cache "+sub.getFeedUrl());
112 feed = feedFetcher.retrieveFeed(feedUrl);
113 SyndFeedInfo feedInfo = feedInfoCache.getFeedInfo(feedUrl);
114 if (feedInfo.getLastModified() != null)
115 {
116 long lastUpdatedLong =
117 ((Long )feedInfo.getLastModified()).longValue();
118 if (lastUpdatedLong != 0)
119 {
120 lastUpdated = new Date (lastUpdatedLong);
121 }
122 }
123 Thread.sleep(100); }
125 catch (Exception e)
126 {
127 logger.warn("ERROR parsing " + sub.getFeedUrl()
128 + " : " + e.getClass().getName() + " : " + e.getMessage());
129 logger.debug(e);
130 continue;
131 }
132 if (lastUpdated!=null && sub.getLastUpdated()!=null)
133 {
134 Calendar feedCal = Calendar.getInstance();
135 feedCal.setTime(lastUpdated);
136
137 Calendar subCal = Calendar.getInstance();
138 subCal.setTime(sub.getLastUpdated());
139
140 if (!feedCal.after(subCal))
141 {
142 if (logger.isDebugEnabled())
143 {
144 String msg = MessageFormat.format(
145 " Skipping ({0} / {1})",
146 new Object [] {
147 lastUpdated, sub.getLastUpdated()});
148 logger.debug(msg);
149 }
150 continue;
151 }
152 }
153 if (feed.getPublishedDate() != null)
154 {
155 sub.setLastUpdated(feed.getPublishedDate());
156 saveSubscription(sub);
157 }
158
159 Calendar cal = Calendar.getInstance();
163 if (sub.getLastUpdated() != null)
164 {
165 cal.setTime(sub.getLastUpdated());
166 }
167 else
168 {
169 cal.setTime(new Date ());
170 cal.add(Calendar.DATE, -1);
171 }
172
173 int count = 0;
175 Iterator entries = feed.getEntries().iterator();
176 while (entries.hasNext())
177 {
178 try
179 {
180 SyndEntry romeEntry = (SyndEntry) entries.next();
181 PlanetEntryData entry =
182 new PlanetEntryData(feed, romeEntry, sub);
183 if (entry.getPublished() == null)
184 {
185 logger.debug(
186 "No published date, assigning fake date for "+feedUrl);
187 entry.setPublished(cal.getTime());
188 }
189 if (entry.getPermalink() == null)
190 {
191 logger.warn("No permalink, rejecting entry from "+feedUrl);
192 }
193 else
194 {
195 saveEntry(entry);
196 newEntries.add(entry);
197 }
198 cal.add(Calendar.DATE, -1);
199 count++;
200 }
201 catch (Exception e)
202 {
203 logger.error("ERROR processing subscription entry", e);
204 }
205 }
206 logger.debug(" Entry count: " + count);
207 if (count > 0)
208 {
209 Iterator entryIter = sub.getEntries().iterator();
210 while (entryIter.hasNext())
211 {
212 deleteEntry((PlanetEntryData)entryIter.next());
213 }
214 sub.purgeEntries();
215 sub.addEntries(newEntries);
216 if (roller != null) roller.commit();
217 }
218 long subEndTime = System.currentTimeMillis();
219 logger.info(" " + count + " - "
220 + ((subEndTime-subStartTime)/1000.0)
221 + " seconds to process (" + count + ") entries of "
222 + sub.getFeedUrl());
223 }
224 clearCachedAggregations();
226
227 long endTime = System.currentTimeMillis();
228 logger.info("--- DONE --- Refreshed entries in "
229 + ((endTime-startTime)/1000.0) + " seconds");
230 }
231
232 public synchronized void clearCachedAggregations()
233 {
234 aggregationsByGroup.purge();
235 topSubscriptionsByGroup.purge();
236 lastUpdatedByGroup.clear();
237 lastUpdated = new Date ();
238 }
239
240 public Date getLastUpdated()
241 {
242 return lastUpdated;
243 }
244
245 public Date getLastUpdated(PlanetGroupData group)
246 {
247 return (Date )lastUpdatedByGroup.get(group);
248 }
249 }
250 |
Popular Tags |
Java API By Example, From Geeks To Geeks.