1 16 17 package org.apache.roller.business.hibernate; 18 19 import com.sun.syndication.feed.synd.SyndEntry; 20 import com.sun.syndication.feed.synd.SyndFeed; 21 import com.sun.syndication.fetcher.FeedFetcher; 22 import com.sun.syndication.fetcher.impl.FeedFetcherCache; 23 import com.sun.syndication.fetcher.impl.HttpURLFeedFetcher; 24 import com.sun.syndication.fetcher.impl.SyndFeedInfo; 25 import java.io.File ; 26 import java.net.URL ; 27 import java.sql.Timestamp ; 28 import java.text.MessageFormat ; 29 import java.util.ArrayList ; 30 import java.util.Calendar ; 31 import java.util.Date ; 32 import java.util.HashMap ; 33 import java.util.Iterator ; 34 import java.util.List ; 35 import java.util.Map ; 36 import java.util.Set ; 37 import java.util.TreeSet ; 38 import org.hibernate.Criteria; 39 import org.hibernate.HibernateException; 40 import org.hibernate.Query; 41 import org.hibernate.Session; 42 import org.hibernate.criterion.Expression; 43 import org.hibernate.criterion.Order; 44 import org.apache.commons.logging.Log; 45 import org.apache.commons.logging.LogFactory; 46 import org.apache.roller.RollerException; 47 import org.apache.roller.config.RollerConfig; 48 import org.apache.roller.config.RollerRuntimeConfig; 49 import org.apache.roller.model.PlanetManager; 50 import org.apache.roller.model.Roller; 51 import org.apache.roller.model.RollerFactory; 52 import org.apache.roller.pojos.PlanetConfigData; 53 import org.apache.roller.pojos.PlanetEntryData; 54 import org.apache.roller.pojos.PlanetGroupData; 55 import org.apache.roller.pojos.PlanetGroupSubscriptionAssoc; 56 import org.apache.roller.pojos.PlanetSubscriptionData; 57 import org.apache.roller.util.rome.DiskFeedInfoCache; 58 59 62 public class HibernatePlanetManagerImpl implements PlanetManager { 63 64 private static Log log = LogFactory.getLog(HibernatePlanetManagerImpl.class); 65 66 protected static final String NO_GROUP = "zzz_nogroup_zzz"; 67 68 private HibernatePersistenceStrategy strategy = null; 69 private String localURL = null; 70 private Map lastUpdatedByGroup = new HashMap (); 71 72 public HibernatePlanetManagerImpl(HibernatePersistenceStrategy strat) { 73 74 this.strategy = strat; 75 76 localURL = RollerRuntimeConfig.getProperty("site.absoluteurl"); 78 } 79 80 public void saveConfiguration(PlanetConfigData config) 81 throws RollerException { 82 strategy.store(config); 83 } 84 85 public void saveGroup(PlanetGroupData group) 86 throws RollerException { 87 88 Iterator assocs = group.getGroupSubscriptionAssocs().iterator(); 90 while (assocs.hasNext()) { 91 PlanetGroupSubscriptionAssoc assoc = 92 (PlanetGroupSubscriptionAssoc)assocs.next(); 93 strategy.store(assoc); 94 } 95 strategy.store(group); 96 } 97 98 public void saveEntry(PlanetEntryData entry) 99 throws RollerException { 100 strategy.store(entry); 101 } 102 103 public void saveSubscription(PlanetSubscriptionData sub) 104 throws RollerException { 105 PlanetSubscriptionData existing = getSubscription(sub.getFeedURL()); 106 if (existing == null || (existing.getId().equals(sub.getId()))) { 107 this.strategy.store(sub); 108 } else { 109 throw new RollerException("ERROR: duplicate feed URLs not allowed"); 110 } 111 } 112 113 public void deleteEntry(PlanetEntryData entry) 114 throws RollerException { 115 strategy.remove(entry); 116 } 117 118 public void deleteGroup(PlanetGroupData group) 119 throws RollerException { 120 strategy.remove(group); 121 } 122 123 public void deleteSubscription(PlanetSubscriptionData sub) 124 throws RollerException { 125 strategy.remove(sub); 126 } 127 128 public PlanetConfigData getConfiguration() throws RollerException { 129 PlanetConfigData config = null; 130 try { 131 Session session = ((HibernatePersistenceStrategy)strategy).getSession(); 132 Criteria criteria = session.createCriteria(PlanetConfigData.class); 133 criteria.setMaxResults(1); 134 List list = criteria.list(); 135 config = list.size()!=0 ? (PlanetConfigData)list.get(0) : null; 136 137 if (config != null) { 140 config.setCacheDir( 141 RollerConfig.getProperty("planet.aggregator.cache.dir")); 142 } 143 } catch (HibernateException e) { 144 throw new RollerException(e); 145 } 146 return config; 147 } 148 149 public PlanetSubscriptionData getSubscription(String feedURL) 150 throws RollerException { 151 try { 152 Session session = ((HibernatePersistenceStrategy)strategy).getSession(); 153 Criteria criteria = 154 session.createCriteria(PlanetSubscriptionData.class); 155 criteria.setMaxResults(1); 156 criteria.add(Expression.eq("feedURL", feedURL)); 157 List list = criteria.list(); 158 return list.size()!=0 ? (PlanetSubscriptionData)list.get(0) : null; 159 } catch (HibernateException e) { 160 throw new RollerException(e); 161 } 162 } 163 164 public PlanetSubscriptionData getSubscriptionById(String id) 165 throws RollerException { 166 return (PlanetSubscriptionData) strategy.load(id, PlanetSubscriptionData.class); 167 } 168 169 public Iterator getAllSubscriptions() { 170 try { 171 Session session = ((HibernatePersistenceStrategy)strategy).getSession(); 172 Criteria criteria = 173 session.createCriteria(PlanetSubscriptionData.class); 174 criteria.addOrder(Order.asc("feedURL")); 175 List list = criteria.list(); 176 return list.iterator(); 177 } catch (Throwable e) { 178 throw new RuntimeException ( 179 "ERROR fetching subscription collection", e); 180 } 181 } 182 183 public int getSubscriptionCount() throws RollerException { 184 try { 185 Session session = ((HibernatePersistenceStrategy)strategy).getSession(); 186 Integer count = (Integer )session.createQuery( 187 "select count(*) from org.apache.roller.pojos.PlanetSubscriptionData").uniqueResult(); 188 return count.intValue(); 189 } catch (Throwable e) { 190 throw new RuntimeException ( 191 "ERROR fetching subscription count", e); 192 } 193 } 194 195 public synchronized List getTopSubscriptions(int offset, int length) 196 throws RollerException { 197 return getTopSubscriptions(null, offset, length); 198 } 199 200 public synchronized List getTopSubscriptions( 201 String groupHandle, int offset, int length) 202 throws RollerException { 203 List ret = null; 204 try { 205 Session session = ((HibernatePersistenceStrategy)strategy).getSession(); 206 Query query = null; 207 if (groupHandle != null) { 208 query = session.createQuery( 209 "select sub from org.apache.roller.pojos.PlanetSubscriptionData sub " 210 +"join sub.groupSubscriptionAssocs assoc " 211 +"where " 212 +"assoc.group.handle=:groupHandle " 213 +"order by sub.inboundblogs desc"); 214 query.setString("groupHandle", groupHandle); 215 } else { 216 query = session.createQuery( 217 "select sub from org.apache.roller.pojos.PlanetSubscriptionData sub " 218 +"order by sub.inboundblogs desc"); 219 } 220 if (offset != 0) { 221 query.setFirstResult(offset); 222 } 223 if (length != -1) { 224 query.setMaxResults(length); 225 } 226 ret = query.list(); 227 } catch (HibernateException e) { 228 throw new RollerException(e); 229 } 230 return ret; 231 } 232 233 public PlanetGroupData getGroup(String handle) throws RollerException { 234 try { 235 Session session = strategy.getSession(); 236 Criteria criteria = session.createCriteria(PlanetGroupData.class); 237 criteria.setMaxResults(1); 238 criteria.add(Expression.eq("handle", handle)); 239 return (PlanetGroupData) criteria.uniqueResult(); 240 } catch (HibernateException e) { 241 throw new RollerException(e); 242 } 243 } 244 245 public PlanetGroupData getGroupById(String id) throws RollerException { 246 return (PlanetGroupData) strategy.load(id, PlanetGroupData.class); 247 } 248 249 public List getGroups() throws RollerException { 250 try { 251 Session session = ((HibernatePersistenceStrategy)strategy).getSession(); 252 Criteria criteria = session.createCriteria(PlanetGroupData.class); 253 return criteria.list(); 254 } catch (HibernateException e) { 255 throw new RollerException(e); 256 } 257 } 258 259 public List getGroupHandles() throws RollerException { 260 List handles = new ArrayList (); 261 Iterator list = getGroups().iterator(); 262 while (list.hasNext()) { 263 PlanetGroupData group = (PlanetGroupData)list.next(); 264 handles.add(group.getHandle()); 265 } 266 return handles; 267 } 268 269 public List getFeedEntries(String feedURL, int offset, int length) 270 throws RollerException { 271 try { 273 Session session = ((HibernatePersistenceStrategy)strategy).getSession(); 274 Criteria criteria = session.createCriteria(PlanetEntryData.class); 275 criteria.add(Expression.eq("subscription.feedURL", feedURL)); 276 criteria.addOrder(Order.desc("pubTime")); 277 criteria.setFirstResult(offset); 278 if (length != -1) criteria.setMaxResults(length); 279 return criteria.list(); 280 } catch (HibernateException e) { 281 throw new RollerException(e); 282 } 283 } 284 285 public synchronized List getAggregation(int offset, int len) 286 throws RollerException { 287 return getAggregation(null, null, null, offset, len); 288 } 289 290 public synchronized List getAggregation(Date startDate, Date endDate, int offset, int len) 291 throws RollerException { 292 return getAggregation(null, startDate, endDate, offset, len); 293 } 294 295 public synchronized List getAggregation(PlanetGroupData group, int offset, int len) 296 throws RollerException { 297 return getAggregation(group, null, null, offset, len); 298 } 299 300 public synchronized List getAggregation( 301 PlanetGroupData group, Date startDate, Date endDate, int offset, int length) 302 throws RollerException { 303 List ret = null; 305 if (endDate == null) endDate = new Date (); 306 try { 307 String groupHandle = (group == null) ? NO_GROUP : group.getHandle(); 308 long startTime = System.currentTimeMillis(); 309 Session session = 310 ((HibernatePersistenceStrategy)strategy).getSession(); 311 312 if (group != null) { 313 StringBuffer sb = new StringBuffer (); 314 sb.append("select entry from org.apache.roller.pojos.PlanetEntryData entry "); 315 sb.append("join entry.subscription.groupSubscriptionAssocs assoc "); 316 sb.append("where assoc.group=:group and entry.pubTime < :endDate "); 317 if (startDate != null) { 318 sb.append("and entry.pubTime > :startDate "); 319 } 320 sb.append("order by entry.pubTime desc"); 321 Query query = session.createQuery(sb.toString()); 322 query.setEntity("group", group); 323 query.setFirstResult(offset); 324 if (length != -1) query.setMaxResults(length); 325 query.setParameter("endDate", endDate); 326 if (startDate != null) { 327 query.setParameter("startDate", startDate); 328 } 329 ret = query.list(); 330 } else { 331 StringBuffer sb = new StringBuffer (); 332 sb.append("select entry from org.apache.roller.pojos.PlanetEntryData entry "); 333 sb.append("join entry.subscription.groupSubscriptionAssocs assoc "); 334 sb.append("where (assoc.group.handle='external' or assoc.group.handle='all') "); 335 sb.append("and entry.pubTime < :endDate "); 336 if (startDate != null) { 337 sb.append("and entry.pubTime > :startDate "); 338 } 339 sb.append("order by entry.pubTime desc"); 340 Query query = session.createQuery(sb.toString()); 341 query.setFirstResult(offset); 342 if (length != -1) query.setMaxResults(length); 343 query.setParameter("endDate", endDate); 344 if (startDate != null) { 345 query.setParameter("startDate", startDate); 346 } 347 ret = query.list(); 348 } 349 Date retLastUpdated = null; 350 if (ret.size() > 0) { 351 PlanetEntryData entry = (PlanetEntryData)ret.get(0); 352 retLastUpdated = entry.getPubTime(); 353 } else { 354 retLastUpdated = new Date (); 355 } 356 lastUpdatedByGroup.put(groupHandle, retLastUpdated); 357 358 long endTime = System.currentTimeMillis(); 359 log.debug("Generated aggregation in " 360 +((endTime-startTime)/1000.0)+" seconds"); 361 362 } catch (Throwable e) { 363 log.error("ERROR: building aggregation for: "+group, e); 364 throw new RollerException(e); 365 } 366 return ret; 367 } 368 369 public synchronized void clearCachedAggregations() { 370 lastUpdatedByGroup.clear(); 371 } 372 373 public Date getLastUpdated() { 374 return (Date )lastUpdatedByGroup.get(NO_GROUP); 375 } 376 377 public Date getLastUpdated(PlanetGroupData group) { 378 return (Date )lastUpdatedByGroup.get(group); 379 } 380 381 public void refreshEntries() throws RollerException { 382 383 Roller roller = RollerFactory.getRoller(); 384 385 Date now = new Date (); 386 long startTime = System.currentTimeMillis(); 387 PlanetConfigData config = getConfiguration(); 388 389 if (config == null || config.getCacheDir() == null) { 391 log.warn("Planet cache directory not set, aborting refresh"); 392 return; 393 } 394 395 String cacheDirName = config.getCacheDir().replaceFirst( 397 "\\$\\{user.home}",System.getProperty("user.home")); 398 399 if (System.getProperty("catalina.home") != null) { 401 cacheDirName = cacheDirName.replaceFirst( 402 "\\$\\{catalina.home}",System.getProperty("catalina.home")); 403 } 404 405 File cacheDir = null; 407 try { 408 cacheDir = new File (cacheDirName); 409 if (!cacheDir.exists()) cacheDir.mkdirs(); 410 } catch (Exception e) { 411 log.error("Unable to create planet cache directory"); 412 return; 413 } 414 415 if (!cacheDir.canWrite()) { 417 log.error("Planet cache directory is not writable"); 418 return; 419 } 420 421 FeedFetcherCache feedInfoCache = 422 new DiskFeedInfoCache(cacheDirName); 423 424 if (config.getProxyHost()!=null && config.getProxyPort() > 0) { 425 System.setProperty("proxySet", "true"); 426 System.setProperty("http.proxyHost", config.getProxyHost()); 427 System.setProperty("http.proxyPort", 428 Integer.toString(config.getProxyPort())); 429 } 430 431 System.setProperty("sun.net.client.defaultConnectTimeout", "15000"); 432 System.setProperty("sun.net.client.defaultReadTimeout", "15000"); 433 434 FeedFetcher feedFetcher = new HttpURLFeedFetcher(feedInfoCache); 435 feedFetcher.setUsingDeltaEncoding(false); 437 feedFetcher.setUserAgent("RollerPlanetAggregator"); 438 439 Iterator subs = getAllSubscriptions(); 441 while (subs.hasNext()) { 442 443 long subStartTime = System.currentTimeMillis(); 444 445 PlanetSubscriptionData sub = (PlanetSubscriptionData)subs.next(); 446 447 sub = this.getSubscriptionById(sub.getId()); 449 450 Set newEntries = this.getNewEntries(sub, feedFetcher, feedInfoCache); 459 int count = newEntries.size(); 460 461 log.debug(" Entry count: " + count); 462 if (count > 0) { 463 sub.purgeEntries(); 464 sub.addEntries(newEntries); 465 this.saveSubscription(sub); 466 if(roller != null) roller.flush(); 467 } 468 long subEndTime = System.currentTimeMillis(); 469 log.info(" " + count + " - " 470 + ((subEndTime-subStartTime)/1000.0) 471 + " seconds to process (" + count + ") entries of " 472 + sub.getFeedURL()); 473 } 474 clearCachedAggregations(); 476 477 long endTime = System.currentTimeMillis(); 478 log.info("--- DONE --- Refreshed entries in " 479 + ((endTime-startTime)/1000.0) + " seconds"); 480 } 481 482 protected Set getNewEntries(PlanetSubscriptionData sub, 483 FeedFetcher feedFetcher, 484 FeedFetcherCache feedInfoCache) 485 throws RollerException { 486 487 Set newEntries = new TreeSet (); 488 SyndFeed feed = null; 489 URL feedURL = null; 490 Date lastUpdated = new Date (); 491 try { 492 feedURL = new URL (sub.getFeedURL()); 493 log.debug("Get feed from cache "+sub.getFeedURL()); 494 feed = feedFetcher.retrieveFeed(feedURL); 495 SyndFeedInfo feedInfo = feedInfoCache.getFeedInfo(feedURL); 496 if (feedInfo.getLastModified() != null) { 497 long lastUpdatedLong = 498 ((Long )feedInfo.getLastModified()).longValue(); 499 if (lastUpdatedLong != 0) { 500 lastUpdated = new Date (lastUpdatedLong); 501 } 502 } 503 Thread.sleep(100); } catch (Exception e) { 505 log.warn("ERROR parsing " + sub.getFeedURL() 506 + " : " + e.getClass().getName() + " : " + e.getMessage()); 507 log.debug(e); 508 return newEntries; } 510 if (lastUpdated!=null && sub.getLastUpdated()!=null) { 511 Calendar feedCal = Calendar.getInstance(); 512 feedCal.setTime(lastUpdated); 513 514 Calendar subCal = Calendar.getInstance(); 515 subCal.setTime(sub.getLastUpdated()); 516 517 if (!feedCal.after(subCal)) { 518 if (log.isDebugEnabled()) { 519 String msg = MessageFormat.format( 520 " Skipping ({0} / {1})", 521 new Object [] { 522 lastUpdated, sub.getLastUpdated()}); 523 log.debug(msg); 524 } 525 return newEntries; } 527 } 528 if (feed.getPublishedDate() != null) { 529 sub.setLastUpdated(feed.getPublishedDate()); 530 } 532 533 Calendar cal = Calendar.getInstance(); 537 if (sub.getLastUpdated() != null) { 538 cal.setTime(sub.getLastUpdated()); 539 } else { 540 cal.setTime(new Date ()); 541 cal.add(Calendar.DATE, -1); 542 } 543 544 Iterator entries = feed.getEntries().iterator(); 546 while (entries.hasNext()) { 547 try { 548 SyndEntry romeEntry = (SyndEntry) entries.next(); 549 PlanetEntryData entry = 550 new PlanetEntryData(feed, romeEntry, sub); 551 if (entry.getPubTime() == null) { 552 log.debug( 553 "No published date, assigning fake date for "+feedURL); 554 entry.setPubTime(new Timestamp (cal.getTimeInMillis())); 555 } 556 if (entry.getPermalink() == null) { 557 log.warn("No permalink, rejecting entry from "+feedURL); 558 } else { 559 newEntries.add(entry); 560 } 561 cal.add(Calendar.DATE, -1); 562 } catch (Exception e) { 563 log.error("ERROR processing subscription entry", e); 564 } 565 } 566 return newEntries; 567 } 568 569 protected String getLocalURL() { 570 return localURL; 571 } 572 573 } 574 575 | Popular Tags |