Java > Open Source Codes > org > apache > roller > business > hibernate > HibernateRefererManagerImpl


1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one or more
3  * contributor license agreements. The ASF licenses this file to You
4  * under the Apache License, Version 2.0 (the "License"); you may not
5  * use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License. For additional information regarding
15  * copyright in this work, please see the NOTICE file in the top level
16  * directory of this distribution.
17  */
18
19 package org.apache.roller.business.hibernate;
20
21 import java.util.ArrayList ;
22 import java.util.Calendar ;
23 import java.util.Date ;
24 import java.util.Iterator ;
25 import java.util.List ;
26 import org.apache.commons.lang.StringUtils;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.hibernate.Criteria;
30 import org.hibernate.Hibernate;
31 import org.hibernate.HibernateException;
32 import org.hibernate.Query;
33 import org.hibernate.Session;
34 import org.hibernate.criterion.Expression;
35 import org.hibernate.criterion.Junction;
36 import org.hibernate.criterion.Order;
37 import org.hibernate.type.Type;
38 import org.apache.roller.RollerException;
39 import org.apache.roller.config.RollerRuntimeConfig;
40 import org.apache.roller.model.RefererManager;
41 import org.apache.roller.pojos.RefererData;
42 import org.apache.roller.pojos.WeblogEntryData;
43 import org.apache.roller.pojos.WebsiteData;
44 import org.apache.roller.pojos.WebsiteDisplayData;
45 import org.hibernate.dialect.OracleDialect;
46 import org.hibernate.dialect.SQLServerDialect;
47 import org.hibernate.engine.SessionFactoryImplementor;
48 import org.hibernate.dialect.Dialect;
49 import org.apache.roller.model.Roller;
50 import org.apache.roller.model.RollerFactory;
51 import org.apache.roller.model.UserManager;
52 import org.apache.roller.model.WeblogManager;
53 import org.apache.roller.pojos.StatCount;
54 import org.apache.roller.util.DateUtil;
55 import org.apache.roller.util.LinkbackExtractor;
56 import org.apache.roller.util.Utilities;
57
58
59 /**
60  * Hibernate implementation of the RefererManager.
61  */
62 public class HibernateRefererManagerImpl implements RefererManager {
63     
64     static final long serialVersionUID = -4966091850482256435L;
65     
66     private static Log log = LogFactory.getLog(HibernateRefererManagerImpl.class);
67     
68     protected static final String DAYHITS = "dayHits";
69     protected static final String TOTALHITS = "totalHits";
70     
71     private HibernatePersistenceStrategy strategy = null;
72     private Date mRefDate = new Date ();
73     
74         
75     public HibernateRefererManagerImpl(HibernatePersistenceStrategy strat) {
76         
77         log.debug("Instantiating Hibernate Referer Manager");
78         
79         strategy = strat;
80     }
81     
82     public void saveReferer(RefererData referer) throws RollerException {
83         strategy.store(referer);
84     }
85         
86     public void removeReferer(RefererData referer) throws RollerException {
87         strategy.remove(referer);
88     }
89         
90     /**
91      * Clear referrer dayhits and remove referrers without excerpts.
92      *
93      * TODO: do we really need dialect specific queries?
94      */
95     public void clearReferrers() throws RollerException {
96         
97         if (log.isDebugEnabled()) {
98             log.debug("clearReferrers");
99         }
100         try {
101             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
102             Dialect currentDialect = ((SessionFactoryImplementor)session.getSessionFactory()).getDialect();
103             String reset = "update RefererData set dayHits=0";
104             session.createQuery(reset).executeUpdate();
105             String delete = null;
106             if ( currentDialect instanceof SQLServerDialect || currentDialect instanceof OracleDialect ){
107                 delete = "delete RefererData where excerpt is null or excerpt like ''";
108             } else {
109                 delete = "delete RefererData where excerpt is null or excerpt=''";
110             }
111             session.createQuery(delete).executeUpdate();
112         } catch (Exception e) {
113             log.error("EXCEPTION resetting referers",e);
114         }
115     }
116         
117     /**
118      * Clear referrer dayhits and remove referrers without excerpts.
119      *
120      * TODO: do we really need dialect specific queries?
121      */
122     public void clearReferrers(WebsiteData website) throws RollerException {
123         
124         if (log.isDebugEnabled()) {
125             log.debug("clearReferrers");
126         }
127         try {
128             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
129             Dialect currentDialect = ((SessionFactoryImplementor)session.getSessionFactory()).getDialect();
130             String reset = "update RefererData set dayHits=0 where website=:site";
131             session.createQuery(reset)
132             .setParameter("site",website).executeUpdate();
133             String delete = null;
134             if ( currentDialect instanceof SQLServerDialect || currentDialect instanceof OracleDialect ){
135                 delete = "delete RefererData where website=:site and (excerpt is null or excerpt like '')";
136             } else {
137                 delete = "delete RefererData where website=:site and (excerpt is null or excerpt='')";
138             }
139             session.createQuery(delete)
140             .setParameter("site",website).executeUpdate();
141         } catch (Exception e) {
142             log.error("EXCEPTION resetting referers",e);
143         }
144     }
145         
146     /**
147      * Apply ignoreWord/spam filters to all referers in system.
148      */
149     public void applyRefererFilters() throws RollerException {
150         
151         try {
152             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
153             Criteria criteria = session.createCriteria(RefererData.class);
154             
155             String spamwords = RollerRuntimeConfig.getProperty("spam.blacklist");
156             
157             String [] blacklist = StringUtils.split(
158                     StringUtils.deleteWhitespace(spamwords),",");
159             Junction or = Expression.disjunction();
160             for (int i=0; i<blacklist.length; i++) {
161                 String ignoreWord = blacklist[i].trim();
162                 //log.debug("including ignore word - "+ignoreWord);
163 or.add(Expression.ilike("refererUrl","%"+ignoreWord+"%"));
164             }
165             criteria.add(Expression.conjunction()
166             .add(Expression.disjunction().add(Expression.isNull("excerpt")).add(Expression.eq("excerpt", "")))
167             .add(or)
168             );
169             
170             log.debug("removing spam referers - "+criteria.list().size());
171             
172             Iterator referer = criteria.list().iterator();
173             while (referer.hasNext()) {
174                 this.strategy.remove((RefererData) referer.next());
175             }
176
177         } catch (HibernateException e) {
178             throw new RollerException(e);
179         }
180     }
181         
182     /**
183      * Apply ignoreWord/spam filters to all referers in website.
184      */
185     public void applyRefererFilters(WebsiteData website) throws RollerException {
186         
187         if (null == website) throw new RollerException("website is null");
188         if (null == website.getBlacklist()) return;
189         
190         try {
191             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
192             Criteria criteria = session.createCriteria(RefererData.class);
193             
194             String [] blacklist = StringUtils.split(
195                     StringUtils.deleteWhitespace(website.getBlacklist()),",");
196             if (blacklist.length == 0) return;
197             
198             Junction or = Expression.disjunction();
199             for (int i=0; i<blacklist.length; i++) {
200                 String ignoreWord = blacklist[i].trim();
201                 or.add(Expression.ilike("refererUrl","%"+ignoreWord+"%"));
202             }
203             criteria.add(Expression.conjunction()
204             .add(Expression.disjunction().add(Expression.isNull("excerpt")).add(Expression.eq("excerpt", "")))
205             .add(Expression.eq("website",website))
206             .add(or)
207             );
208             
209             Iterator referer = criteria.list().iterator();
210             while (referer.hasNext()) {
211                 this.strategy.remove((RefererData) referer.next());
212             }
213             
214         } catch (HibernateException e) {
215             throw new RollerException(e);
216         }
217     }
218     
219     /**
220      * Use Hibernate directly because Roller's Query API does too much allocation.
221      */
222     protected List getExistingReferers(WebsiteData website, String dateString,
223             String permalink) throws RollerException {
224         
225         try {
226             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
227             Criteria criteria = session.createCriteria(RefererData.class);
228             criteria.add(Expression.conjunction()
229             .add(Expression.eq("website",website))
230             .add(Expression.eq("dateString",dateString))
231             .add(Expression.eq("refererPermalink",permalink)));
232             
233             return criteria.list();
234         } catch (HibernateException e) {
235             throw new RollerException(e);
236         }
237     }
238         
239     /**
240      * Use Hibernate directly because Roller's Query API does too much allocation.
241      */
242     protected List getMatchingReferers(WebsiteData website, String requestUrl,
243             String refererUrl) throws RollerException {
244         
245         try {
246             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
247             Criteria criteria = session.createCriteria(RefererData.class);
248             criteria.add(Expression.conjunction()
249             .add(Expression.eq("website",website))
250             .add(Expression.eq("requestUrl",requestUrl))
251             .add(Expression.eq("refererUrl",refererUrl)));
252             
253             return criteria.list();
254         } catch (HibernateException e) {
255             throw new RollerException(e);
256         }
257     }
258        
259     /**
260      * Returns hot weblogs as StatCount objects, in descending order by today's hits.
261      */
262     public List getHotWeblogs(int sinceDays, int offset, int length)
263         throws RollerException {
264         // TODO: ATLAS getDaysPopularWebsites DONE TESTED
265 String msg = "Getting hot weblogs";
266         ArrayList result = new ArrayList ();
267         Calendar cal = Calendar.getInstance();
268         cal.setTime(new Date ());
269         cal.add(Calendar.DATE, -1 * sinceDays);
270         Date startDate = cal.getTime();
271         try {
272             Session session =
273                 ((HibernatePersistenceStrategy)strategy).getSession();
274             Query query = session.createQuery(
275                 "select sum(r.dayHits) as s, w.id, w.name, w.handle "
276                +"from WebsiteData w, RefererData r "
277                +"where r.website=w and w.enabled=true and w.active=true and w.lastModified > :startDate "
278                +"group by w.name, w.handle, w.id order by col_0_0_ desc");
279             query.setParameter("startDate", startDate);
280             
281               // +"group by w.name, w.handle, w.id order by s desc");
282 // The above would be *much* better but "HQL parser does not
283 // resolve alias in ORDER BY clause" (See Hibernate issue HHH-892)
284
285             if (offset != 0) {
286                 query.setFirstResult(offset);
287             }
288             if (length != -1) {
289                 query.setMaxResults(length);
290             }
291             Iterator rawResults = query.list().iterator();
292             for (Iterator it = query.list().iterator(); it.hasNext();) {
293                 Object [] row = (Object [])it.next();
294                 Integer hits = (Integer )row[0];
295                 String websiteId = (String )row[1];
296                 String websiteName = (String )row[2];
297                 String websiteHandle = (String )row[3];
298                 result.add(new StatCount(
299                     websiteId,
300                     websiteHandle,
301                     websiteName,
302                     "statCount.weblogDayHits",
303                     hits.longValue()));
304             }
305             return result;
306             
307         } catch (Throwable pe) {
308             log.error(msg, pe);
309             throw new RollerException(msg, pe);
310         }
311     }
312     
313     /**
314      * @deprecated Replaced by getHotWeblogs().
315      */
316     public List getDaysPopularWebsites(int offset, int length)
317         throws RollerException {
318         // TODO: ATLAS getDaysPopularWebsites DONE TESTED
319 String msg = "Getting popular websites";
320         ArrayList result = new ArrayList ();
321         try {
322             Session session =
323                 ((HibernatePersistenceStrategy)strategy).getSession();
324             Query query = session.createQuery(
325                 "select sum(r.dayHits) as s, w.id, w.name, w.handle "
326                +"from WebsiteData w, RefererData r "
327                +"where r.website=w and w.enabled=true and w.active=true "
328                +"group by w.name, w.handle, w.id order by col_0_0_ desc");
329             
330               // +"group by w.name, w.handle, w.id order by s desc");
331 // The above would be *much* better but "HQL parser does not
332 // resolve alias in ORDER BY clause" (See Hibernate issue HHH-892)
333
334             if (offset != 0) {
335                 query.setFirstResult(offset);
336             }
337             if (length != -1) {
338                 query.setMaxResults(length);
339             }
340             Iterator rawResults = query.list().iterator();
341             for (Iterator it = query.list().iterator(); it.hasNext();) {
342                 Object [] row = (Object [])it.next();
343                 Integer hits = (Integer )row[0];
344                 String websiteId = (String )row[1];
345                 String websiteName = (String )row[2];
346                 String websiteHandle = (String )row[3];
347                 result.add(new WebsiteDisplayData(
348                     websiteId,
349                     websiteName,
350                     websiteHandle,
351                     hits));
352             }
353             return result;
354             
355         } catch (Throwable pe) {
356             log.error(msg, pe);
357             throw new RollerException(msg, pe);
358         }
359     }
360         
361     /**
362      * Use raw SQL because Hibernate can't handle the query.
363      */
364     protected int getHits(WebsiteData website, String type) throws RollerException {
365         int hits = 0;
366         if (log.isDebugEnabled()) {
367             log.debug("getHits: " + website.getName());
368         }
369         
370         Object [] args = { Boolean.TRUE, website.getId() };
371         Type[] types = { Hibernate.BOOLEAN, Hibernate.STRING };
372         
373         // For a query like this, Hibernate returns a list of lists
374 Session session = ((HibernatePersistenceStrategy)strategy).getSession();
375         List results;
376         try {
377             // begin transaction
378 this.strategy.getSession().beginTransaction();
379             
380             Query q = session.createQuery(
381                     "select sum(h.dayHits),sum(h.totalHits) from h in class " +
382                     "org.apache.roller.pojos.RefererData " +
383                     "where h.website.enabled=? and h.website.id=? ");
384             q.setParameters(args, types);
385             results = q.list();
386         } catch (HibernateException e) {
387             throw new RollerException(e);
388         }
389         Object [] resultsArray = (Object []) results.get(0);
390         
391         if (resultsArray.length > 0 && type.equals(DAYHITS)) {
392             if ( resultsArray[0] != null ) {
393                 hits = ((Integer ) resultsArray[0]).intValue();
394             }
395         } else if ( resultsArray.length > 0 ) {
396             if ( resultsArray[0] != null ) {
397                 hits = ((Integer ) resultsArray[1]).intValue();
398             }
399         } else {
400             hits = 0;
401         }
402         
403         return hits;
404     }
405         
406     /**
407      * @see org.apache.roller.pojos.RefererManager#getReferers(java.lang.String)
408      */
409     public List getReferers(WebsiteData website) throws RollerException {
410         if (website==null )
411             throw new RollerException("website is null");
412         
413         try {
414             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
415             Criteria criteria = session.createCriteria(RefererData.class);
416             criteria.add(Expression.eq("website",website));
417             criteria.addOrder(Order.desc("totalHits"));
418             
419             return criteria.list();
420         } catch (HibernateException e) {
421             throw new RollerException(e);
422         }
423     }
424         
425     /**
426      * @see org.apache.roller.pojos.RefererManager#getTodaysReferers(String)
427      */
428     public List getTodaysReferers(WebsiteData website) throws RollerException {
429         if (website==null )
430             throw new RollerException("website is null");
431         
432         try {
433             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
434             Criteria criteria = session.createCriteria(RefererData.class);
435             criteria.add(Expression.eq("website", website));
436             criteria.add(Expression.gt("dayHits", new Integer (0)));
437             criteria.addOrder(Order.desc("dayHits"));
438             
439             return criteria.list();
440         } catch (HibernateException e) {
441             throw new RollerException(e);
442         }
443     }
444        
445     /**
446      * Returns referers for a specified day. Duplicate enties are not
447      * included in this list so the hit counts may not be accurate.
448      * @see org.apache.roller.pojos.RefererManager#getReferersToDate(
449      * org.apache.roller.pojos.WebsiteData, java.lang.String)
450      */
451     public List getReferersToDate(WebsiteData website, String date)
452             throws RollerException {
453         if (website==null )
454             throw new RollerException("website is null");
455         
456         if (date==null )
457             throw new RollerException("Date is null");
458         
459         try {
460             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
461             Criteria criteria = session.createCriteria(RefererData.class);
462             criteria.add(Expression.eq("website", website));
463             criteria.add(Expression.eq("dateString", date));
464             criteria.add(Expression.eq("duplicate", Boolean.FALSE));
465             criteria.addOrder(Order.desc("totalHits"));
466             
467             return criteria.list();
468         } catch (HibernateException e) {
469             throw new RollerException(e);
470         }
471     }
472         
473     /**
474      * @see org.apache.roller.pojos.RefererManager#getReferersToEntry(
475      * java.lang.String, java.lang.String)
476      */
477     public List getReferersToEntry(String entryid) throws RollerException {
478         if (null == entryid)
479             throw new RollerException("entryid is null");
480         
481         try {
482             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
483             Criteria criteria = session.createCriteria(RefererData.class);
484             criteria.createAlias("weblogEntry","e");
485             
486             criteria.add(Expression.eq("e.id", entryid));
487             criteria.add(Expression.isNotNull("title"));
488             criteria.add(Expression.isNotNull("excerpt"));
489             
490             criteria.addOrder(Order.desc("totalHits"));
491             
492             return criteria.list();
493         } catch (HibernateException e) {
494             throw new RollerException(e);
495         }
496     }
497         
498     /**
499      * Query for collection of referers.
500      */
501     protected List getReferersToWebsite(WebsiteData website, String refererUrl)
502             throws RollerException {
503         
504         try {
505             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
506             Criteria criteria = session.createCriteria(RefererData.class);
507             criteria.add(Expression.eq("website", website));
508             criteria.add(Expression.eq("refererUrl", refererUrl));
509             
510             return criteria.list();
511         } catch (HibernateException e) {
512             throw new RollerException(e);
513         }
514     }
515         
516     /**
517      * Query for collection of referers.
518      */
519     protected List getReferersWithSameTitle(WebsiteData website,
520                                             String requestUrl,
521                                             String title,
522                                             String excerpt)
523             throws RollerException {
524         
525         try {
526             Session session = ((HibernatePersistenceStrategy)strategy).getSession();
527             Criteria criteria = session.createCriteria(RefererData.class);
528             
529             Junction conjunction = Expression.conjunction();
530             conjunction.add(Expression.eq("website", website));
531             conjunction.add(Expression.eq("requestUrl", requestUrl));
532             
533             Junction disjunction = Expression.conjunction();
534             disjunction.add(Expression.eq("title", title));
535             disjunction.add(Expression.eq("excerpt", excerpt));
536             
537             criteria.add(conjunction);
538             criteria.add(disjunction);
539             
540             return criteria.list();
541         } catch (HibernateException e) {
542             throw new RollerException(e);
543         }
544     }
545         
546     public int getDayHits(WebsiteData website) throws RollerException {
547         return getHits(website, DAYHITS);
548     }
549         
550     public int getTotalHits(WebsiteData website) throws RollerException {
551         return getHits(website, TOTALHITS);
552     }
553         
554     /**
555      * @see org.apache.roller.pojos.RefererManager#retrieveReferer(java.lang.String)
556      */
557     public RefererData getReferer(String id) throws RollerException {
558         return (RefererData)strategy.load(id,RefererData.class);
559     }
560     
561     
562     public void processReferrer(String requestUrl,
563                                 String referrerUrl,
564                                 String weblogHandle,
565                                 String entryAnchor,
566                                 String dateString) {
567         
568         log.debug("processing referrer ["+referrerUrl+
569                 "] accessing ["+requestUrl+"]");
570         
571         if (weblogHandle == null)
572             return;
573         
574         String selfSiteFragment = "/"+weblogHandle;
575         WebsiteData weblog = null;
576         WeblogEntryData entry = null;
577         
578         // lookup the weblog now
579 try {
580             UserManager userMgr = RollerFactory.getRoller().getUserManager();
581             weblog = userMgr.getWebsiteByHandle(weblogHandle);
582             if (weblog == null) return;
583             
584             // now lookup weblog entry if possible
585 if (entryAnchor != null) {
586                 WeblogManager weblogMgr = RollerFactory.getRoller().getWeblogManager();
587                 entry = weblogMgr.getWeblogEntryByAnchor(weblog, entryAnchor);
588             }
589         } catch (RollerException re) {
590             // problem looking up website, gotta bail
591 log.error("Error looking up website object", re);
592             return;
593         }
594         
595         try {
596             List matchRef = null;
597             
598             // try to find existing RefererData for referrerUrl
599 if (referrerUrl == null || referrerUrl.trim().length() < 8) {
600                 referrerUrl = "direct";
601                 
602                 // Get referer specified by referer URL of direct
603 matchRef = getReferersToWebsite(weblog, referrerUrl);
604             } else {
605                 referrerUrl = Utilities.stripJsessionId(referrerUrl);
606                 
607                 // Query for referer with same referer and request URLs
608 matchRef = getMatchingReferers(weblog, requestUrl, referrerUrl);
609                 
610                 // If referer was not found, try adding or leaving off 'www'
611 if ( matchRef.size() == 0 ) {
612                     String secondTryUrl = null;
613                     if ( referrerUrl.startsWith("http://www") ) {
614                         secondTryUrl = "http://"+referrerUrl.substring(11);
615                     } else {
616                         secondTryUrl = "http://www"+referrerUrl.substring(7);
617                     }
618                     
619                     matchRef = getMatchingReferers(weblog, requestUrl, secondTryUrl);
620                     if ( matchRef.size() == 1 ) {
621                         referrerUrl = secondTryUrl;
622                     }
623                 }
624             }
625             
626             if (matchRef.size() == 1) {
627                 // Referer was found in database, so bump up hit count
628 RefererData ref = (RefererData)matchRef.get(0);
629                 
630                 ref.setDayHits(new Integer (ref.getDayHits().intValue() + 1));
631                 ref.setTotalHits(new Integer (ref.getTotalHits().intValue() + 1));
632                 
633                 log.debug("Incrementing hit count on existing referer: "+referrerUrl);
634                 
635                 saveReferer(ref);
636                 
637             } else if (matchRef.size() == 0) {
638                 
639                 // Referer was not found in database, so new Referer object
640 Integer one = new Integer (1);
641                 RefererData ref =
642                         new RefererData(
643                         null,
644                         weblog,
645                         entry,
646                         dateString,
647                         referrerUrl,
648                         null,
649                         requestUrl,
650                         null,
651                         "", // Read comment above regarding Derby bug
652 Boolean.FALSE,
653                         Boolean.FALSE,
654                         one,
655                         one);
656                 
657                 if (log.isDebugEnabled()) {
658                     log.debug("newReferer="+ref.getRefererUrl());
659                 }
660                 
661                 String refurl = ref.getRefererUrl();
662                 
663                 // If not a direct or search engine then search for linkback
664 boolean doLinkbackExtraction =
665                         RollerRuntimeConfig.getBooleanProperty("site.linkbacks.enabled");
666                 if (doLinkbackExtraction
667                         && entry != null
668                         && !refurl.equals("direct")
669                         && !refurl.startsWith("http://google")
670                         && !refurl.startsWith("http://www.google")
671                         && !refurl.startsWith("http://search.netscape")
672                         && !refurl.startsWith("http://www.blinkpro")
673                         && !refurl.startsWith("http://search.msn")
674                         && !refurl.startsWith("http://search.yahoo")
675                         && !refurl.startsWith("http://uk.search.yahoo")
676                         && !refurl.startsWith("http://www.javablogs.com")
677                         && !refurl.startsWith("http://www.teoma")
678                         ) {
679                     // Launch thread to extract referer linkback
680
681                     try {
682                         Roller mRoller = RollerFactory.getRoller();
683                         mRoller.getThreadManager().executeInBackground(
684                                 new LinkbackExtractorRunnable(ref));
685                     } catch (InterruptedException e) {
686                         log.warn("Interrupted during linkback extraction",e);
687                     }
688                 } else {
689                     saveReferer(ref);
690                 }
691             }
692         } catch (RollerException pe) {
693             log.error(pe);
694         } catch (NullPointerException npe) {
695             log.error(npe);
696         }
697     }
698         
699     /**
700      * Use LinkbackExtractor to parse title and excerpt from referer
701      */
702     class LinkbackExtractorRunnable implements Runnable {
703         
704         private RefererData mReferer = null;
705         
706         public LinkbackExtractorRunnable( RefererData referer) {
707             mReferer = referer;
708         }
709         
710         public void run() {
711             
712             try {
713                 LinkbackExtractor lb = new LinkbackExtractor(
714                         mReferer.getRefererUrl(),mReferer.getRequestUrl());
715                 
716                 if ( lb.getTitle()!=null && lb.getExcerpt()!=null ) {
717                     mReferer.setTitle(lb.getTitle());
718                     mReferer.setExcerpt(lb.getExcerpt());
719                     
720                     if ( lb.getPermalink() != null ) {
721                         // The presence of a permalink indicates that this
722 // linkback was parsed out of an RSS feed and is
723 // presumed to be a good linkback.
724
725                         mReferer.setRefererPermalink(lb.getPermalink());
726                         
727                         // See if this request/permalink is in the DB
728 List matchRef = getExistingReferers(
729                                 mReferer.getWebsite(),
730                                 mReferer.getDateString(),
731                                 mReferer.getRefererPermalink());
732                         
733                         // If it is the first, then set it to be visible
734 if ( matchRef.size() == 0 ) {
735                             mReferer.setVisible(Boolean.TRUE);
736                         } else {
737                             // We can't throw away duplicates or we will
738 // end up reparsing them everytime a hit comes
739 // in from one of them, but we can mark them
740 // as duplicates.
741 mReferer.setDuplicate(Boolean.TRUE);
742                         }
743                         
744                         saveReferer(mReferer);
745                         
746                     }
747                     
748                     else {
749                         // Store the new referer
750 saveReferer(mReferer);
751                         
752                         // Hacky Referer URL weighting kludge:
753 //
754 // If there are multple referers to a request URL,
755 // then we want to pick the best one. The others
756 // are marked as duplicates. To do this we use a
757 // weight. The weight formula is:
758 //
759 // w = URL length + (100 if URL contains anchor)
760
761                         // LOOP: find the referer with the highest weight
762 Boolean visible = Boolean.FALSE;
763                         List refs= getReferersWithSameTitle(
764                                 mReferer.getWebsite(),
765                                 mReferer.getRequestUrl(),
766                                 lb.getTitle(),
767                                 lb.getExcerpt());
768                         RefererData chosen = null;
769                         int maxweight = 0;
770                         for (Iterator rdItr = refs.iterator();rdItr.hasNext();) {
771                             RefererData referer = (RefererData) rdItr.next();
772                             
773                             int weight = referer.getRefererUrl().length();
774                             if (referer.getRefererUrl().indexOf('#') != -1) {
775                                 weight += 100;
776                             }
777                             
778                             if ( weight > maxweight ) {
779                                 chosen = referer;
780                                 maxweight = weight;
781                             }
782                             
783                             if (referer.getVisible().booleanValue()) {
784                                 // If any are visible then chosen
785 // replacement must be visible as well.
786 visible = Boolean.TRUE;
787                             }
788                             
789                         }
790                         
791                         // LOOP: to mark all of the lower weight ones
792 // as duplicates
793 for (Iterator rdItr = refs.iterator();rdItr.hasNext();) {
794                             RefererData referer = (RefererData) rdItr.next();
795                             
796                             if (referer != chosen) {
797                                 referer.setDuplicate(Boolean.TRUE);
798                             } else {
799                                 referer.setDuplicate(Boolean.FALSE);
800                                 referer.setVisible(visible);
801                             }
802                             saveReferer(referer);
803                         }
804                         
805                         
806                     }
807                 } else {
808                     // It is not a linkback, but store it anyway
809 saveReferer(mReferer);
810                     
811                     log.info("No excerpt found at refering URL "
812                             + mReferer.getRefererUrl());
813                 }
814             } catch (Exception e) {
815                 log.error("Processing linkback",e);
816             } finally {
817                 strategy.release();
818             }
819             
820         }
821         
822     }
823        
824     public void release() {}
825 }
826
827
828
829

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags