HttpConnection


1   /*
2    * Copyright 1999,2004 The Apache Software Foundation.
3    * 
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * 
8    *      http://www.apache.org/licenses/LICENSE-2.0
9    * 
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  
17  package org.apache.taglibs.scrape;
18  
19  import java.net.*;
20  import java.io.*;
21  import java.util.*;
22  import java.security.KeyStore  ;
23  import javax.servlet.jsp.PageContext  ;
24  //import javax.net.ssl.SSLSocket;
25  //import com.sun.net.ssl.SSLContext;
26  //import com.sun.net.ssl.KeyManagerFactory;
27  //import javax.net.ssl.SSLSocketFactory;
28  //import javax.net.ssl.HandshakeCompletedEvent;
29  //import javax.net.ssl.HandshakeCompletedListener;
30  
31  /**
32   * HttpConnection - the class that creates the http connection that the rest of the
33   *                  package uses 
34   *
35   * @author Rich Catlett
36   *
37   * @version 1.0
38   *
39   */
40  public class HttpConnection {
41  
42      /**
43       * the URL of this connection
44       */
45      private URL url = null;
46      /**
47       * socket to make the connection on
48       */
49      private Socket socket = null;
50      /**
51       * hashmap to hold http response headers
52       */
53      private HashMap keys = null;
54      /**
55       * set holds the set of keys for the hashmap keys
56       */
57      private Vector headers = new Vector(12, 5);
58      /**
59       * value to determine if headers have been retrieved or not
60       */
61      private boolean gotten = false;
62      /**
63       * separator in the response headers list
64       */
65      private static final char keySeparator = ':';
66      /**
67       * flag tells if this object is making it's http connection through proxies
68       */
69      private boolean useproxy = false;
70      /**
71       * the port to use for the proxy connection
72       */
73      private int pport = 3128;
74      /**
75       * the proxy server to use for the connection
76       */
77      private String   pserver = null;
78      /**
79       * error from proxy server when connection failed
80       */
81      private String   proxyerror = null;
82      /**
83       * boolean value determines if the connection to to travel via a secure
84       * connection
85       */
86      //private boolean ssl = false;
87      /**
88       * base64 encoded username and password used for proxy authentication
89       */
90      private String   auth = null;
91      /**
92       * outputstream for this connection
93       */
94      private OutputStream out = null;
95      /**
96       * the port for this connection will default to 80 for http and 443 for https
97       */
98      private int port;
99      /**
100      * internet address for this connection
101      */
102     private InetAddress dst = null;
103     /**
104      * the password to the client keystore for ssl authentication
105      */
106     private String   sslpass = null;
107     /**
108      * response code from the request
109      */
110     private int responseCode = 0;
111     /**
112      * response message from the request
113      */
114     private String   responseMessage = null;
115     /**
116      * the request method HEAD or GET
117      */
118     private String   requestMethod = "GET";
119     /**
120      * flag determines if the the class is connected
121      */
122     private boolean connected = false;
123     /**
124      * pageContext of the servlet used for logging
125      */
126     private PageContext   pageContext = null;
127     /**
128      * pagedata object that represents the page to be scraped
129      */
130     private PageData pagedata = null;
131 
132     /**
133      * constructor creates an instance of HttpConnection and calls the super class
134      * HttpURLConnection
135      *
136      * @param url - url of the http server to connect to
137      *
138      */
139     public HttpConnection(URL url, PageData pd, PageContext   pc) {
140     //public HttpConnection(URL url, boolean secure, String pass, PageContext pc) {
141     this.url = url;
142     //ssl = secure;
143     //sslpass = pass;
144     pageContext = pc;
145         pagedata = pd;
146         if (pagedata.getProxyServer() != null) {
147             useproxy = true;
148     }
149     }
150 
151     /**
152      * constructor creates an instance of HttpConnection and calls the super
153      * class HttpURLConnection, this instance of the class connects through a 
154      * proxy server that requires authentication
155      *
156      * @param url  url of the http server to connect to
157      * @param port  the port to use for the connection to the proxy server
158      * @param server  the proxy server
159      * @param secure  is the link going over https
160      * @param name  user name for authentication
161      * @param pass  password for authentication
162      *
163      */
164     public HttpConnection(URL url, int port, String   server, String   authstring, 
165     PageContext   pc) {
166     //public HttpConnection(URL url, int port, String server, boolean secure, 
167     //String pass, String authstring, PageContext pc) {
168         this.url = url;
169         pport = port;
170         pserver = server;
171         useproxy = true;
172     //ssl = secure;
173     //sslpass = pass;
174     pageContext = pc;
175     if (authstring != null)
176         auth = "Basic " + 
177         new sun.misc.BASE64Encoder().encode(authstring.getBytes());
178     }
179 
180     /**
181      * Implementation of the abstract method defined in the URLConnection class
182      * establish a connection to an HTTP server and send request
183      *
184      * @throws IOException - if connection cannot be made
185      *
186      */
187     public void connect() throws IOException {
188 
189     if (connected) // connected is defined in URLConnection
190         return;
191 
192     // get port number
193     if ((port = getURL().getPort()) == -1) {
194         if (pagedata.getSSL())
195         port = 443;  // default port for https
196             else
197             port = 80;   // default port for http
198     }
199 
200     // create the socket for this connection
201     //if (!pagedata.getSSL()) {
202         // make a connection for http
203         if (useproxy) {
204         // create socket to the proxy server
205         InetAddress proxy = InetAddress.getByName(pagedata.getProxyServer());
206         // make the connection to the proxy server
207         socket = new Socket(proxy, pagedata.getProxyPort());
208         } else {
209         // create the Internet address object for this url
210         InetAddress dst = InetAddress.getByName(getURL().getHost());
211         // create a regular socket
212         socket = new Socket(dst, port);
213         }
214         // get the outputstream
215         out = socket.getOutputStream();
216         //} else
217         // make a connection for https
218         //makeSecureConnection();
219 
220     // set timeout value for this connection
221     try {
222         socket.setSoTimeout(20000);
223     } catch (SocketException se) {
224         pageContext.getServletContext().
225         log("timeout could not be set on the socket");
226     }
227     connected = true;  // set value of connected
228     }
229 
230     /**
231      * Implementation of the abstract method defined in the HttpURLConnection
232      * class cut the current connection this object has
233      *
234      */
235     public void disconnect() {
236     try {
237         socket.close();  // close the connection
238         socket = null;
239     } catch (IOException ie) {}
240 
241     // reset values on the connection
242     out = null;
243     gotten = false;
244     keys = null;
245 
246     connected = false;  // set value of connected
247     }
248 
249     /**
250      * Implementation of the abstract method defined in the HttpURLConnection
251      * class this implementation does not use proxy
252      *
253      * @return  false proxy servers not used by this implementation
254      *
255      */
256     public boolean usingProxy() {
257     return useproxy;
258     }
259 
260     /**
261      * Override the default method provided in the URLConnection class
262      *
263      */
264     public InputStream getInputStream() throws IOException {
265     if (!connected)
266         connect();
267 
268     return socket.getInputStream();
269     }
270 
271     /**
272      * Override the default method provided in the URLConnection class
273      *
274      */
275     public OutputStream getOutputStream() throws IOException {
276     if (!connected)
277         connect();
278 
279     return socket.getOutputStream();
280     }
281 
282     /**
283      * get the request method
284      *
285      * @return - the request method
286      *
287      */
288     public String   getRequestMethod() {
289     return requestMethod;
290     }
291 
292     /**
293      * set the request method
294      *
295      * @param String  GET or HEAD
296      *
297      */
298     public void setRequestMethod(String   value) {
299     requestMethod = value;
300     }
301 
302     /**
303      * Override the default method provided in the HttpURLConnection class
304      * Get the response code for this connection
305      *
306      * @return - the response code of the connection, -1 if no valid response
307      *
308      */
309     public int getResponseCode() {
310     return responseCode;
311     }
312 
313     /**
314      * Override the default method provided in the HttpURLConnection class
315      * Get the response message for the last request
316      *
317      * @return - the response message of the last request
318      *
319      */
320     public String   getResponseMessage() {
321     return responseMessage;
322     }
323 
324     /**
325      * send the request to the server
326      *
327      * @throws IOException
328      *
329      */
330     public void sendRequest() throws IOException {
331         ArrayList headers = pagedata.getHeaders();
332     //System.out.println("sendRequest() useproxy = " + useproxy + "\n" + 
333     //      getRequestMethod() + " " + url.toString() + " HTTP/1.1\r\n");
334     // send request to the server via http check for not ssl because if a proxy
335     // and ssl are in use the request is to be tunneled through the proxy
336     // connection and is already setup
337     if (useproxy && !pagedata.getSSL()) {
338         send(out, getRequestMethod() + " " + url.toString() + " HTTP/1.1\r\n");
339         if (pagedata.getAuth() != null)
340         send(out, "Proxy-Authorization: " + pagedata.getAuth() + "\r\n");
341             if (headers != null) {
342                 ArrayList name = (ArrayList) headers.get(0);
343                 ArrayList value = (ArrayList) headers.get(1);
344                 for (int i = 0; i < name.size(); i++) {
345                     send(out, name.get(i) + ": " + value.get(i) + "\r\n");
346         }
347         }
348         send(out, "Connection: close\r\n");
349         } else {
350         send(out, getRequestMethod() + " " + url.getFile() + " HTTP/1.1\r\n");
351         send(out, "Host: " + url.getHost() + "\r\n");
352             if (headers != null) {
353                 ArrayList name = (ArrayList) headers.get(0);
354                 ArrayList value = (ArrayList) headers.get(1);
355                 for (int i = 0; i < name.size(); i++) {
356                     send(out, name.get(i) + ": " + value.get(i) + "\r\n");
357         }
358         }
359     }
360     //send(sslout, "Accept: text/plain, text/html \r\n");
361     send(out, "\r\n");
362 
363     // get the response message to the request
364     responseMessage = recv(socket.getInputStream());
365     try {
366         responseCode = Integer.parseInt(responseMessage.substring(9, 12));
367     } catch (NumberFormatException   nfe) {responseCode = -1;}
368 
369     // check for and take care of redirection
370     if (responseCode == 301 || responseCode == 302)
371         doRedirection();
372     }
373 
374     /**
375      * take care of redirection if the response code is 301 or 302
376      *
377      */
378     private void doRedirection() throws IOException {
379     setURL(getHeaderField("Location"));
380     // log the redirection
381     pageContext.getServletContext().
382            log(getResponseMessage() + ": Connection redirected to " +
383                getHeaderField("Location"));
384     // close connection so that a new one can be opened to Location:
385     disconnect();
386     connect();
387     sendRequest();
388     }
389 
390 
391     /**
392      * make a regular http connection
393      *
394      * @throws IOException
395      *
396      */
397     boolean makeTunnelConnection() throws IOException {
398     //System.out.println("makeTunnelConnection() sending CONNECT to proxy CONNECT "
399     //         + url.getHost() + ":" + port + " HTTP/1.1");
400   
401     // create socket to the proxy server
402     InetAddress proxy = InetAddress.getByName(pagedata.getProxyServer());
403     // make the connection to the proxy server
404     socket = new Socket(proxy, pagedata.getProxyPort());
405 
406     // get the outputstream
407     out = socket.getOutputStream();
408 
409     send(out, "CONNECT " + url.getHost() + ":" + port + " HTTP/1.0\r\n");
410     if (pagedata.getAuth() != null) {
411       //System.out.println("sending the auth line Proxy-Authorization: " + auth);
412         send(out, "Proxy-Authorization: " + pagedata.getAuth() + "\r\n");
413     }
414     send(out, "\r\n");
415 
416     // get the response from the proxy server for the connection
417     String   sb = recv(socket.getInputStream());
418 
419     if (sb.startsWith("HTTP/1.0 200"))
420         // connection successful
421         return true;
422         else {
423         // connection failed
424         proxyerror = sb.substring(9);
425         return false;
426     }
427     }
428 
429     /**
430      * get the sslsocket
431      *
432      * @return - the created sslsocket
433      *
434      * @throws IOException
435      *
436      *
437     private void makeSecureConnection() throws IOException {
438     System.out.println("makeSecureConnection()");
439     if (useproxy) {
440         // going to tunnel the ssl connection through the proxy
441         if (!makeTunnelConnection()) {
442         // throw exception for some reason
443             throw new IOException("Connection to the proxy server could not be "
444                       + "extablished: " + proxyerror);
445         }
446     }
447     SSLSocket sslsocket = null;
448     try {
449         SSLSocketFactory sslfactory;
450         SSLContext context = SSLContext.getInstance("TLS");
451         // if client authorization is required for the ssl connection init the
452         // SSLContext with the clients keystore
453         if (sslpass != null) {
454         System.out.println("using client authentication for ssl");
455         KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
456             FileInputStream filein = 
457          new FileInputStream(System.getProperty("javax.net.ssl.trustStore"));
458         KeyStore ks = KeyStore.getInstance("JKS");
459         ks.load((InputStream)filein, null);
460         System.out.println("password = " + sslpass);
461         char[] password = pagedata.getSslPass.toCharArray();
462         kmf.init(ks, password);
463         context.init(kmf.getKeyManagers(), null, null);
464                 sslfactory = context.getSocketFactory();
465         } else
466         // use the default SSLSocketFactory
467         sslfactory = (SSLSocketFactory) SSLSocketFactory.getDefault();
468 
469         if (!useproxy)
470         sslsocket = (SSLSocket) sslfactory.createSocket(dst, port);
471             else {
472                 sslsocket = 
473                    (SSLSocket) sslfactory.createSocket(socket, url.getHost(), 
474                                port, true);
475         //System.out.println("created the tunneled socket");
476         }
477     } catch (java.security.GeneralSecurityException nsae) {
478         // should never happen
479         throw new IOException("HttpConnection.connect(): " + 
480                 "creating tunneled sslsocket " + nsae.toString());
481     }
482     System.out.println("starting sslhandshake");
483     sslsocket.startHandshake();
484     socket = sslsocket;
485     out = sslsocket.getOutputStream();
486     System.out.println("makeSecureConnection() DONE");
487     }
488     */
489     /**
490      * Override default provided in URLConnection
491      *
492      * @param n - index of the desired header field
493      *
494      */
495     public String   getHeaderField(int n) {
496         getHeaders();  // get headers from http response
497 
498         if (n < headers.size()) {  // make sure n is not out of bounds
499             return getField((String  )headers.elementAt(n));  // return header field
500         }
501         return null;
502     }
503 
504     /**
505      * Override default provided in URLConnection
506      *
507      * @return - LastModified header or 0 if LastModified does not exist
508      *
509      */
510     public long getLastModified() {
511         getHeaders();  // get headers from http response
512 
513         String   header = (String  )keys.get("LastModified");
514         if (header != null) {
515             try {
516             return new Integer  (header).longValue();
517             } catch (NumberFormatException   nfe) {
518             return 0;
519             }
520         }
521     return 0;
522     }
523 
524     /**
525      * Override default provided in URLConnection
526      *
527      * @return - the Expiration header or 0 if Expiration does not exist
528      *
529      */
530     public long getExpiration() {
531         getHeaders();  // get headers from http response
532 
533     String   header = (String  )keys.get("Expiration");
534         if (header != null) {
535             try {
536             return new Integer  (header).longValue();
537             } catch (NumberFormatException   nfe) {
538             return 0;
539             }
540         }
541     return 0;
542     }
543 
544     /**
545      * Override default provided in URLConnection
546      *
547      * @param key - name of the desired header field
548      *
549      */
550     public String   getHeaderField(String   key) {
551         getHeaders();  // get headers from http response
552 
553         return (String  )keys.get(key.toLowerCase());  // return desired header field
554     }
555 
556     /**
557      * Override default provided in URLConnection
558      *
559      * @param n   value returned if header is null
560      * @param key  header to retreive
561      *
562      * @return - header as an integer
563      */
564     public int getHeaderFieldInt(String   key, int n) {
565         getHeaders();  // get headers from http response
566 
567     String   header = (String  )keys.get(key.toLowerCase());
568         if (header != null) {
569         try {
570         n = new Integer  (header).intValue();
571         } catch (NumberFormatException   nfe) {
572         // do nothing the default value specified will be returned
573         }
574     }
575     return n;
576     }
577 
578     /**
579      * Override default provided in URLConnection
580      *
581      * @param n - index of the desired header field
582      *
583      */
584     public String   getHeaderFieldKey(int n) {
585         getHeaders();  // get headers from http response
586 
587         if (n < headers.size()) {  // make sure n is not out of bounds
588         // return name of nth header field
589             return getKey((String  )headers.elementAt(n));
590         }
591         return null;
592     }
593 
594     /**
595      * Set the URL for this connection
596      *
597      * @throws MalformedURLException
598      *
599      */
600     protected void setURL(String   value) throws MalformedURLException {
601     url = new URL(value);
602     }
603 
604     /**
605      * get the URL of this connection
606      *
607      * @return - the URL for this connection
608      */
609     public URL getURL() {
610     return url;
611     }
612 
613     /**
614      * Helper routine for parsing header field
615      *
616      * @param str - complete header field name and value separated by :
617      *
618      * @return - name of header field
619      *
620      */
621     private String   getKey(String   str) {
622         if (str == null)
623             return null;
624         int ind = str.indexOf(keySeparator);
625         if (ind >= 0)
626             return str.substring(0, ind).toLowerCase();
627         return null;
628     }
629 
630     /**
631      * Helper routine for parsing header field
632      *
633      * @param str - complete header field name and value separated by :
634      *
635      * @return value of header field
636      *
637      */
638     private String   getField(String   str) {
639         if (str == null)
640             return null;
641         int ind = str.indexOf(keySeparator);
642         if (ind >= 0)
643             return str.substring(ind+1).trim();
644         else
645             return str;
646     }
647 
648     /**
649      * Helper routine that reads header from HTTP connection
650      *
651      */
652     private void getHeaders() {
653 
654     String   header;  // holds one full header name value pair
655 
656     // check if headers have been gotten previously
657         if (gotten)
658             return;
659         gotten = true;  // set gotten
660 
661     // initialize keys
662     keys = new HashMap();
663 
664         try {
665             connect();  // make connection
666 
667         // get inputstream from connection
668             InputStream in = getInputStream();
669 
670         header = recv(in);  // get initial header field
671 
672             while (true) {
673                 if (header.length() == 0) // exit loop if no more headers to parse
674                     break;
675         headers.addElement(header);   // build the vector headers
676                 String   key = getKey(header);  // parse the header name
677                 if (key != null) {
678                     keys.put(key, getField(header));  // parse the header value
679                 }
680                 header = recv(in);  // get next header
681             }
682         } catch (IOException e) {
683             e.printStackTrace();
684         }
685     }
686 
687     /**
688      * helper routine to send output stream over the http connection
689      *
690      * @param msg - String sent via out to http server
691      * @param out - OutputStream msg is written to
692      *
693      * @throws IOException when a problem occurs with the write
694      */
695     private void send(OutputStream out, String   msg) throws IOException {
696     // create outputstreamwriter to write whole String
697     OutputStreamWriter output = new OutputStreamWriter(out);
698 
699     output.write(msg, 0, msg.length());  // write to output stream
700     output.flush();
701     }
702 
703     /**
704      * Helper routine to read a newline-terminated string from input stream
705      *
706      * @param in - inputstream to read from
707      *
708      * @throws IOException when a problem occurs with the read
709      *
710      */
711     private String   recv(InputStream in) throws IOException {
712         String   result = "";  // final result from read to be returned
713         int c = in.read();   // read in initial char from inputstream
714 
715         while (c >= 0 && c != '\n') {
716             if (c != '\r') {
717                 result += (char)c;
718             }
719             c = in.read();
720         }
721     //System.out.println("The results of the read = " + result);
722         return result;
723     }
724 }
725
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags