Java > Open Source Codes > com > gargoylesoftware > htmlunit > HttpWebConnection


1 /*
2  * Copyright (c) 2002, 2005 Gargoyle Software Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions are met:
6  *
7  * 1. Redistributions of source code must retain the above copyright notice,
8  * this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright notice,
10  * this list of conditions and the following disclaimer in the documentation
11  * and/or other materials provided with the distribution.
12  * 3. The end-user documentation included with the redistribution, if any, must
13  * include the following acknowledgment:
14  *
15  * "This product includes software developed by Gargoyle Software Inc.
16  * (http://www.GargoyleSoftware.com/)."
17  *
18  * Alternately, this acknowledgment may appear in the software itself, if
19  * and wherever such third-party acknowledgments normally appear.
20  * 4. The name "Gargoyle Software" must not be used to endorse or promote
21  * products derived from this software without prior written permission.
22  * For written permission, please contact info@GargoyleSoftware.com.
23  * 5. Products derived from this software may not be called "HtmlUnit", nor may
24  * "HtmlUnit" appear in their name, without prior written permission of
25  * Gargoyle Software Inc.
26  *
27  * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
28  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
29  * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GARGOYLE
30  * SOFTWARE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
31  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
33  * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
36  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37  */
38 package com.gargoylesoftware.htmlunit;
39
40 import java.io.ByteArrayInputStream ;
41 import java.io.IOException ;
42 import java.io.InputStream ;
43 import java.io.UnsupportedEncodingException ;
44 import java.net.URL ;
45 import java.util.ArrayList ;
46 import java.util.HashMap ;
47 import java.util.Iterator ;
48 import java.util.List ;
49 import java.util.Map ;
50
51 import org.org.apache.commons.httpclient.Header;
52 import org.org.apache.commons.httpclient.HostConfiguration;
53 import org.org.apache.commons.httpclient.HttpClient;
54 import org.org.apache.commons.httpclient.HttpException;
55 import org.org.apache.commons.httpclient.HttpMethod;
56 import org.org.apache.commons.httpclient.HttpMethodBase;
57 import org.org.apache.commons.httpclient.HttpMethodRetryHandler;
58 import org.org.apache.commons.httpclient.HttpState;
59 import org.org.apache.commons.httpclient.HttpStatus;
60 import org.org.apache.commons.httpclient.NameValuePair;
61 import org.org.apache.commons.httpclient.URI;
62 import org.org.apache.commons.httpclient.URIException;
63 import org.org.apache.commons.httpclient.auth.CredentialsProvider;
64 import org.org.apache.commons.httpclient.methods.GetMethod;
65 import org.org.apache.commons.httpclient.methods.PostMethod;
66 import org.org.apache.commons.httpclient.methods.StringRequestEntity;
67 import org.org.apache.commons.httpclient.methods.multipart.FilePart;
68 import org.org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity;
69 import org.org.apache.commons.httpclient.methods.multipart.Part;
70 import org.org.apache.commons.httpclient.methods.multipart.StringPart;
71 import org.org.apache.commons.httpclient.params.HttpMethodParams;
72 import org.apache.commons.io.IOUtils;
73 import org.apache.commons.logging.Log;
74 import org.apache.commons.logging.LogFactory;
75 import org.apache.commons.logging.impl.SimpleLog;
76
77 /**
78  * An object that handles the actual communication portion of page
79  * retrieval/submission <p />
80  *
81  * THIS CLASS IS FOR INTERNAL USE ONLY
82  *
83  * @version $Revision: 100 $
84  * @author <a HREF="mailto:mbowler@GargoyleSoftware.com">Mike Bowler</a>
85  * @author Noboru Sinohara
86  * @author David D. Kilzer
87  * @author Marc Guillemot
88  */
89 public class HttpWebConnection extends WebConnection {
90     private final Map httpClients_ = new HashMap ( 9 );
91
92     /**
93      * Create an instance that will not use a proxy server
94      *
95      * @param webClient The WebClient that is using this connection
96      */
97     public HttpWebConnection( final WebClient webClient ) {
98         super(webClient);
99     }
100
101
102     /**
103      * Create an instance that will use the specified proxy server
104      *
105      * @param proxyHost The server that will act as proxy
106      * @param proxyPort The port to use on the proxy server
107      * @param webClient The web client that is using this connection
108      */
109     public HttpWebConnection( final WebClient webClient, final String proxyHost, final int proxyPort ) {
110         super(webClient, proxyHost, proxyPort);
111     }
112
113
114     /**
115      * Submit a request and retrieve a response
116      *
117      * @param webRequestSettings Settings to make the request with
118      * @return See above
119      * @exception IOException If an IO error occurs
120      */
121     public WebResponse getResponse(final WebRequestSettings webRequestSettings) throws IOException {
122
123         final URL url = webRequestSettings.getURL();
124         final FormEncodingType encType = webRequestSettings.getEncodingType();
125         final SubmitMethod submitMethod = webRequestSettings.getSubmitMethod();
126         final List parameters = webRequestSettings.getRequestParameters();
127         final String body = webRequestSettings.getRequestBody();
128         final Map requestHeaders = webRequestSettings.getAdditionalHeaders();
129         final CredentialsProvider credentialsProvider = webRequestSettings.getCredentialsProvider();
130
131         final HttpClient httpClient = getHttpClientFor( url );
132
133         try {
134             final HttpMethod httpMethod = makeHttpMethod( url, encType, submitMethod, parameters, body,
135                 requestHeaders, credentialsProvider );
136             final long startTime = System.currentTimeMillis();
137             final int responseCode = httpClient.executeMethod( httpMethod );
138             final long endTime = System.currentTimeMillis();
139             return makeWebResponse( responseCode, httpMethod, url, endTime-startTime );
140         }
141         catch( final HttpException e ) {
142             // KLUDGE: hitting www.yahoo.com will cause an exception to be thrown while
143 // www.yahoo.com/ (note the trailing slash) will not. If an exception is
144 // caught here then check to see if this is the situation. If so, then retry
145 // it with a trailing slash. The bug manifests itself with httpClient
146 // complaining about not being able to find a line with HTTP/ on it.
147 if( url.getPath().length() == 0 ) {
148                 final StringBuffer buffer = new StringBuffer ();
149                 buffer.append(url.getProtocol());
150                 buffer.append("://");
151                 buffer.append(url.getHost());
152                 buffer.append("/");
153                 if( url.getQuery() != null ) {
154                     buffer.append(url.getQuery());
155                 }
156                 //TODO: There might be a bug here since the original encoding type is lost.
157 final WebRequestSettings newRequest = new WebRequestSettings(new URL (buffer.toString()));
158                 newRequest.setSubmitMethod(submitMethod);
159                 newRequest.setRequestParameters(parameters);
160                 newRequest.setAdditionalHeaders(requestHeaders);
161                 return getResponse(newRequest);
162             }
163             else {
164                 e.printStackTrace();
165                 throw new RuntimeException ( "HTTP Error: " + e.getMessage() );
166             }
167         }
168     }
169
170     /**
171      * Creates an <tt>HttpMethod</tt> instance according to the specified parameters.
172      * @param url The URL which the method is to hit.
173      * @param encType The form encoding type whith the method is to use.
174      * @param method The type of method that is to be returned.
175      * @param parameters The parameters that the returned method is to submit to the web server.
176      * @param body The request body that the method is to send to the server. Used only for POST requests.
177      * @param requestHeaders The request headers that the returned method is to submit to the web server.
178      * @param credentialsProvider The custom credentials provider (if any) that the method is to use.
179      * @return The <tt>HttpMethod</tt> instance constructed according to the specified parameters.
180      * @throws IOException
181      */
182     private HttpMethod makeHttpMethod(
183             final URL url,
184             final FormEncodingType encType,
185             final SubmitMethod method,
186             final List parameters,
187             final String body,
188             final Map requestHeaders,
189             final CredentialsProvider credentialsProvider)
190         throws
191             IOException {
192
193         final HttpMethod httpMethod;
194         String path = url.getPath();
195         if( path.length() == 0 ) {
196             path = "/";
197         }
198         if( method == SubmitMethod.GET ) {
199             httpMethod = new GetMethod( path );
200             if( parameters.isEmpty() ) {
201                 final String queryString = url.getQuery();
202                 httpMethod.setQueryString( queryString );
203             }
204             else {
205                 final NameValuePair[] pairs = new NameValuePair[parameters.size()];
206                 parameters.toArray( pairs );
207                 httpMethod.setQueryString( pairs );
208             }
209         }
210         else if( method == SubmitMethod.POST ) {
211             httpMethod = new PostMethod( path );
212             final String queryString = url.getQuery();
213             if( queryString != null ) {
214                 httpMethod.setQueryString(queryString);
215             }
216             if( body != null ) {
217                 ( (PostMethod) httpMethod ).setRequestEntity( new StringRequestEntity( body ) );
218             }
219             Iterator iterator;
220
221             // Note that this has to be done in two loops otherwise it won't
222 // be able to support two elements with the same name.
223 iterator = parameters.iterator();
224             if (encType == FormEncodingType.URL_ENCODED) {
225                 while( iterator.hasNext() ) {
226                     final NameValuePair pair = ( NameValuePair )iterator.next();
227                     ( ( PostMethod )httpMethod ).removeParameter( pair.getName(), pair.getValue() );
228                 }
229
230                 iterator = parameters.iterator();
231                 while( iterator.hasNext() ) {
232                     final NameValuePair pair = ( NameValuePair )iterator.next();
233                     ( ( PostMethod )httpMethod ).addParameter( pair.getName(), pair.getValue() );
234                 }
235             }
236             else {
237                 final List partList = new ArrayList ();
238                 iterator = parameters.iterator();
239                 while (iterator.hasNext()) {
240                     final Part newPart;
241                     final KeyValuePair pair = (KeyValuePair) iterator.next();
242                     if (pair instanceof KeyDataPair) {
243                         final KeyDataPair pairWithFile = (KeyDataPair) pair;
244                         newPart = new FilePart(
245                                 pairWithFile.getName(),
246                                 pairWithFile.getValue(),
247                                 pairWithFile.getFile(),
248                                 pairWithFile.getContentType(),
249                                 pairWithFile.getCharset());
250                     }
251                     else {
252                         newPart = new StringPart(pair.getName(), pair.getValue());
253                     }
254                     partList.add(newPart);
255                 }
256                 Part[] parts = new Part[partList.size()];
257                 parts = (Part[]) partList.toArray(parts);
258                 ((PostMethod) httpMethod).setRequestEntity(new MultipartRequestEntity(
259                         parts,
260                         httpMethod.getParams()));
261             }
262         }
263         else {
264             throw new IllegalStateException ( "Submit method not yet supported: " + method );
265         }
266
267         httpMethod.setRequestHeader(
268                 "User-Agent", getWebClient().getBrowserVersion().getUserAgent() );
269
270         writeRequestHeadersToHttpMethod( httpMethod, requestHeaders );
271         httpMethod.setFollowRedirects(false);
272         // http://jakarta.apache.org/commons/httpclient/3.0/exception-handling.html#Automatic%20exception%20recovery
273 final HttpMethodRetryHandler noAutoRetry = new HttpMethodRetryHandler() {
274             public boolean retryMethod(final HttpMethod arg0, final IOException arg1, final int arg2) {
275                 return false;
276             }
277         };
278
279         httpMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, noAutoRetry);
280         if(credentialsProvider != null) {
281             httpMethod.getParams().setParameter(CredentialsProvider.PROVIDER, credentialsProvider);
282         }
283         return httpMethod;
284     }
285
286
287     private synchronized HttpClient getHttpClientFor( final URL url ) {
288         final String key = url.getProtocol() + "://" + url.getHost().toLowerCase();
289
290         HttpClient client = ( HttpClient )httpClients_.get( key );
291         if( client == null ) {
292             client = new HttpClient();
293
294             // Disable informational messages from httpclient
295 final Log log = LogFactory.getLog("httpclient.wire");
296             if( log instanceof SimpleLog ) {
297                 ((SimpleLog)log).setLevel( SimpleLog.LOG_LEVEL_WARN );
298             }
299
300             // Tell the client where to get its credentials from.
301 client.getParams().setParameter( CredentialsProvider.PROVIDER, getWebClient().getCredentialsProvider() );
302
303             final HostConfiguration hostConfiguration = new HostConfiguration();
304             final URI uri;
305             try {
306                 uri = new URI(url.toExternalForm(), false);
307             }
308             catch( final URIException e ) {
309                 // Theoretically impossible but ....
310 throw new IllegalStateException ("Unable to create URI from URL: "+url.toExternalForm());
311             }
312             hostConfiguration.setHost(uri);
313             if( getProxyHost() != null ) {
314                 hostConfiguration.setProxy( getProxyHost(), getProxyPort() );
315             }
316             client.setHostConfiguration(hostConfiguration);
317             final int timeout = getWebClient().getTimeout();
318             client.getHttpConnectionManager().getParams().setSoTimeout(timeout);
319             client.getHttpConnectionManager().getParams().setConnectionTimeout(timeout);
320
321             // If two clients are part of the same domain then they should share the same
322 // state (ie cookies)
323 final HttpState sharedState = getStateForUrl( url );
324             if( sharedState != null ) {
325                 client.setState(sharedState);
326             }
327             httpClients_.put( key, client );
328         }
329         return client;
330     }
331
332
333
334     /**
335      * Return the log object for this class
336      * @return The log object
337      */
338     protected final Log getLog() {
339         return LogFactory.getLog(getClass());
340     }
341
342
343     /**
344      * Return the {@link HttpState} that is being used for a given domain
345      * @param url The url from which the domain will be determined
346      * @return The state or null if no state can be found for this domain.
347      */
348     public synchronized HttpState getStateForUrl( final URL url ) {
349         final String domain = url.getHost().toLowerCase();
350         int index = domain.lastIndexOf('.');
351         if( index != -1 ) {
352             index = domain.lastIndexOf(".", index-1);
353         }
354         final String rootDomain;
355         if( index == -1 ) {
356             rootDomain = domain;
357         }
358         else {
359             rootDomain = domain.substring(index+1);
360         }
361
362         final Iterator iterator = httpClients_.entrySet().iterator();
363         while( iterator.hasNext() ) {
364             final Map.Entry entry = (Map.Entry )iterator.next();
365             final String key = (String )entry.getKey();
366             final String host = key.substring(key.indexOf("://") + 3);
367             if( host.equals(rootDomain) || host.endsWith("."+rootDomain) ) {
368                 return ((HttpClient)entry.getValue()).getState();
369             }
370         }
371
372         return null;
373     }
374
375
376     private WebResponse makeWebResponse(
377         final int statusCode, final HttpMethod method, final URL originatingURL, final long loadTime )
378         throws IOException {
379
380         // determine charset
381 final String contentCharSet;
382         if (method instanceof HttpMethodBase) {
383             contentCharSet = ((HttpMethodBase) method).getResponseCharSet();
384         }
385         else {
386             contentCharSet = "ISO-8859-1";
387         }
388
389         // HttpMethod.getResponseBodyAsStream may return null if no body is available
390 final InputStream bodyStream = method.getResponseBodyAsStream();
391         final String content;
392         if (bodyStream == null) {
393             content = "";
394         }
395         else {
396             content = IOUtils.toString(bodyStream, contentCharSet);
397         }
398
399         return new WebResponse() {
400             private String content_ = content;
401             private String contentCharSet_ = contentCharSet;
402
403             public int getStatusCode() {
404                 return statusCode;
405             }
406
407             public String getStatusMessage() {
408                 String message = method.getStatusText();
409                 if( message == null || message.length() == 0 ) {
410                     message = HttpStatus.getStatusText( statusCode );
411                 }
412
413                 if( message == null ) {
414                     message = "Unknown status code";
415                 }
416
417                 return message;
418             }
419
420             public String getContentType() {
421                 final Header contentTypeHeader = method.getResponseHeader( "content-type" );
422                 if( contentTypeHeader == null ) {
423                     // Not technically legal but some servers don't return a content-type
424 return "";
425                 }
426                 final String contentTypeHeaderLine = contentTypeHeader.getValue();
427                 final int index = contentTypeHeaderLine.indexOf( ';' );
428                 if( index == -1 ) {
429                     return contentTypeHeaderLine;
430                 }
431                 else {
432                     return contentTypeHeaderLine.substring( 0, index );
433                 }
434             }
435
436             public String getContentAsString() {
437                 return content_;
438             }
439
440             public InputStream getContentAsStream() {
441                 return new ByteArrayInputStream (getResponseBody());
442             }
443
444             public URL getUrl() {
445                 return originatingURL;
446             }
447
448             public List getResponseHeaders() {
449                 final List headers = new ArrayList ();
450                 final Header[] array = method.getResponseHeaders();
451                 for( int i = 0; i < array.length; i++ ) {
452                     headers.add( new NameValuePair( array[i].getName(), array[i].getValue() ) );
453                 }
454                 return headers;
455             }
456
457             public String getResponseHeaderValue( final String headerName ) {
458                 final Header header = method.getResponseHeader(headerName);
459                 if( header == null ) {
460                     return null;
461                 }
462                 else {
463                     return header.getValue();
464                 }
465             }
466
467             public long getLoadTimeInMilliSeconds() {
468                 return loadTime;
469             }
470
471             public String getContentCharSet(){
472                 return contentCharSet_;
473             }
474
475             public byte [] getResponseBody() {
476                 try {
477                     return content_.getBytes(getContentCharSet());
478                 }
479                 catch (final UnsupportedEncodingException e) {
480                     // should never occur
481 throw new RuntimeException (e);
482                 }
483             }
484
485         };
486     }
487
488
489     private void writeRequestHeadersToHttpMethod( final HttpMethod httpMethod, final Map requestHeaders ) {
490         synchronized( requestHeaders ) {
491             final Iterator iterator = requestHeaders.entrySet().iterator();
492             while( iterator.hasNext() ) {
493                 final Map.Entry entry = ( Map.Entry )iterator.next();
494                 httpMethod.setRequestHeader( ( String )entry.getKey(), ( String )entry.getValue() );
495             }
496         }
497     }
498
499 }
500
501

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags