KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > webharvest > runtime > processors > HttpProcessor


1 /* Copyright (c) 2006-2007, Vladimir Nikic
2     All rights reserved.
3
4     Redistribution and use of this software in source and binary forms,
5     with or without modification, are permitted provided that the following
6     conditions are met:
7
8     * Redistributions of source code must retain the above
9       copyright notice, this list of conditions and the
10       following disclaimer.
11
12     * Redistributions in binary form must reproduce the above
13       copyright notice, this list of conditions and the
14       following disclaimer in the documentation and/or other
15       materials provided with the distribution.
16
17     * The name of Web-Harvest may not be used to endorse or promote
18       products derived from this software without specific prior
19       written permission.
20
21     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22     AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23     IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24     ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
25     LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26     CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27     SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28     INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30     ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31     POSSIBILITY OF SUCH DAMAGE.
32
33     You can contact Vladimir Nikic by sending e-mail to
34     nikic_vladimir@yahoo.com. Please include the word "Web-Harvest" in the
35     subject line.
36 */
37 package org.webharvest.runtime.processors;
38
39 import java.io.UnsupportedEncodingException JavaDoc;
40 import java.util.*;
41
42 import org.apache.commons.httpclient.NameValuePair;
43 import org.webharvest.definition.HttpDef;
44 import org.webharvest.exception.HttpException;
45 import org.webharvest.runtime.Scraper;
46 import org.webharvest.runtime.ScraperContext;
47 import org.webharvest.runtime.scripting.ScriptEngine;
48 import org.webharvest.runtime.templaters.BaseTemplater;
49 import org.webharvest.runtime.variables.*;
50 import org.webharvest.runtime.web.HttpClientManager;
51 import org.webharvest.runtime.web.HttpResponseWrapper;
52
53 /**
54  * Http processor.
55  */
56 public class HttpProcessor extends BaseProcessor {
57
58     private HttpDef httpDef;
59     
60     List httpParams = new ArrayList();
61     Map httpHeaderMap = new HashMap();
62
63     public HttpProcessor(HttpDef httpDef) {
64         super(httpDef);
65         this.httpDef = httpDef;
66     }
67
68     public IVariable execute(Scraper scraper, ScraperContext context) {
69         scraper.setRunningHttpProcessor(this);
70
71         ScriptEngine scriptEngine = scraper.getScriptEngine();
72         String JavaDoc url = BaseTemplater.execute( httpDef.getUrl(), scriptEngine);
73         String JavaDoc method = BaseTemplater.execute( httpDef.getMethod(), scriptEngine);
74         String JavaDoc charset = BaseTemplater.execute( httpDef.getCharset(), scriptEngine);
75         String JavaDoc username = BaseTemplater.execute( httpDef.getUsername(), scriptEngine);
76         String JavaDoc password = BaseTemplater.execute( httpDef.getPassword(), scriptEngine);
77         
78         if (charset == null) {
79             charset = scraper.getConfiguration().getCharset();
80         }
81         
82         // executes body of HTTP processor
83 executeBody(httpDef, scraper, context);
84
85         HttpClientManager manager = scraper.getHttpClientManager();
86
87         HttpResponseWrapper res = manager.execute(method, url, charset, username, password, httpParams, httpHeaderMap);
88
89         scraper.removeRunningHttpProcessor();
90
91         String JavaDoc mimeType = res.getMimeType();
92
93         log.info("Downloaded: " + url + ", mime type = " + mimeType + ", length = " + res.getBody().length + "B.");
94
95         IVariable result;
96         
97         if (mimeType == null || mimeType.toLowerCase().indexOf("text") == 0) {
98             String JavaDoc text;
99             try {
100                 text = new String JavaDoc(res.getBody(), charset);
101             } catch (UnsupportedEncodingException JavaDoc e) {
102                 throw new HttpException("Charset " + charset + " is not supported!", e);
103             }
104             
105             result = new NodeVariable(text);
106         } else {
107             result = new NodeVariable( res.getBody() );
108         }
109         
110         return result;
111     }
112     
113     protected void addHttpParam(String JavaDoc name, String JavaDoc value) {
114         httpParams.add( new NameValuePair(name, value) );
115     }
116     
117     protected void addHttpHeader(String JavaDoc name, String JavaDoc value) {
118         httpHeaderMap.put(name, value);
119     }
120
121 }
Popular Tags