KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > apache > taglibs > scrape > HttpConnection


1 /*
2  * Copyright 1999,2004 The Apache Software Foundation.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */

16
17 package org.apache.taglibs.scrape;
18
19 import java.net.*;
20 import java.io.*;
21 import java.util.*;
22 import java.security.KeyStore JavaDoc;
23 import javax.servlet.jsp.PageContext JavaDoc;
24 //import javax.net.ssl.SSLSocket;
25
//import com.sun.net.ssl.SSLContext;
26
//import com.sun.net.ssl.KeyManagerFactory;
27
//import javax.net.ssl.SSLSocketFactory;
28
//import javax.net.ssl.HandshakeCompletedEvent;
29
//import javax.net.ssl.HandshakeCompletedListener;
30

31 /**
32  * HttpConnection - the class that creates the http connection that the rest of the
33  * package uses
34  *
35  * @author Rich Catlett
36  *
37  * @version 1.0
38  *
39  */

40 public class HttpConnection {
41
42     /**
43      * the URL of this connection
44      */

45     private URL url = null;
46     /**
47      * socket to make the connection on
48      */

49     private Socket socket = null;
50     /**
51      * hashmap to hold http response headers
52      */

53     private HashMap keys = null;
54     /**
55      * set holds the set of keys for the hashmap keys
56      */

57     private Vector headers = new Vector(12, 5);
58     /**
59      * value to determine if headers have been retrieved or not
60      */

61     private boolean gotten = false;
62     /**
63      * separator in the response headers list
64      */

65     private static final char keySeparator = ':';
66     /**
67      * flag tells if this object is making it's http connection through proxies
68      */

69     private boolean useproxy = false;
70     /**
71      * the port to use for the proxy connection
72      */

73     private int pport = 3128;
74     /**
75      * the proxy server to use for the connection
76      */

77     private String JavaDoc pserver = null;
78     /**
79      * error from proxy server when connection failed
80      */

81     private String JavaDoc proxyerror = null;
82     /**
83      * boolean value determines if the connection to to travel via a secure
84      * connection
85      */

86     //private boolean ssl = false;
87
/**
88      * base64 encoded username and password used for proxy authentication
89      */

90     private String JavaDoc auth = null;
91     /**
92      * outputstream for this connection
93      */

94     private OutputStream out = null;
95     /**
96      * the port for this connection will default to 80 for http and 443 for https
97      */

98     private int port;
99     /**
100      * internet address for this connection
101      */

102     private InetAddress dst = null;
103     /**
104      * the password to the client keystore for ssl authentication
105      */

106     private String JavaDoc sslpass = null;
107     /**
108      * response code from the request
109      */

110     private int responseCode = 0;
111     /**
112      * response message from the request
113      */

114     private String JavaDoc responseMessage = null;
115     /**
116      * the request method HEAD or GET
117      */

118     private String JavaDoc requestMethod = "GET";
119     /**
120      * flag determines if the the class is connected
121      */

122     private boolean connected = false;
123     /**
124      * pageContext of the servlet used for logging
125      */

126     private PageContext JavaDoc pageContext = null;
127     /**
128      * pagedata object that represents the page to be scraped
129      */

130     private PageData pagedata = null;
131
132     /**
133      * constructor creates an instance of HttpConnection and calls the super class
134      * HttpURLConnection
135      *
136      * @param url - url of the http server to connect to
137      *
138      */

139     public HttpConnection(URL url, PageData pd, PageContext JavaDoc pc) {
140     //public HttpConnection(URL url, boolean secure, String pass, PageContext pc) {
141
this.url = url;
142     //ssl = secure;
143
//sslpass = pass;
144
pageContext = pc;
145         pagedata = pd;
146         if (pagedata.getProxyServer() != null) {
147             useproxy = true;
148     }
149     }
150
151     /**
152      * constructor creates an instance of HttpConnection and calls the super
153      * class HttpURLConnection, this instance of the class connects through a
154      * proxy server that requires authentication
155      *
156      * @param url url of the http server to connect to
157      * @param port the port to use for the connection to the proxy server
158      * @param server the proxy server
159      * @param secure is the link going over https
160      * @param name user name for authentication
161      * @param pass password for authentication
162      *
163      */

164     public HttpConnection(URL url, int port, String JavaDoc server, String JavaDoc authstring,
165     PageContext JavaDoc pc) {
166     //public HttpConnection(URL url, int port, String server, boolean secure,
167
//String pass, String authstring, PageContext pc) {
168
this.url = url;
169         pport = port;
170         pserver = server;
171         useproxy = true;
172     //ssl = secure;
173
//sslpass = pass;
174
pageContext = pc;
175     if (authstring != null)
176         auth = "Basic " +
177         new sun.misc.BASE64Encoder().encode(authstring.getBytes());
178     }
179
180     /**
181      * Implementation of the abstract method defined in the URLConnection class
182      * establish a connection to an HTTP server and send request
183      *
184      * @throws IOException - if connection cannot be made
185      *
186      */

187     public void connect() throws IOException {
188
189     if (connected) // connected is defined in URLConnection
190
return;
191
192     // get port number
193
if ((port = getURL().getPort()) == -1) {
194         if (pagedata.getSSL())
195         port = 443; // default port for https
196
else
197             port = 80; // default port for http
198
}
199
200     // create the socket for this connection
201
//if (!pagedata.getSSL()) {
202
// make a connection for http
203
if (useproxy) {
204         // create socket to the proxy server
205
InetAddress proxy = InetAddress.getByName(pagedata.getProxyServer());
206         // make the connection to the proxy server
207
socket = new Socket(proxy, pagedata.getProxyPort());
208         } else {
209         // create the Internet address object for this url
210
InetAddress dst = InetAddress.getByName(getURL().getHost());
211         // create a regular socket
212
socket = new Socket(dst, port);
213         }
214         // get the outputstream
215
out = socket.getOutputStream();
216         //} else
217
// make a connection for https
218
//makeSecureConnection();
219

220     // set timeout value for this connection
221
try {
222         socket.setSoTimeout(20000);
223     } catch (SocketException se) {
224         pageContext.getServletContext().
225         log("timeout could not be set on the socket");
226     }
227     connected = true; // set value of connected
228
}
229
230     /**
231      * Implementation of the abstract method defined in the HttpURLConnection
232      * class cut the current connection this object has
233      *
234      */

235     public void disconnect() {
236     try {
237         socket.close(); // close the connection
238
socket = null;
239     } catch (IOException ie) {}
240
241     // reset values on the connection
242
out = null;
243     gotten = false;
244     keys = null;
245
246     connected = false; // set value of connected
247
}
248
249     /**
250      * Implementation of the abstract method defined in the HttpURLConnection
251      * class this implementation does not use proxy
252      *
253      * @return false proxy servers not used by this implementation
254      *
255      */

256     public boolean usingProxy() {
257     return useproxy;
258     }
259
260     /**
261      * Override the default method provided in the URLConnection class
262      *
263      */

264     public InputStream getInputStream() throws IOException {
265     if (!connected)
266         connect();
267
268     return socket.getInputStream();
269     }
270
271     /**
272      * Override the default method provided in the URLConnection class
273      *
274      */

275     public OutputStream getOutputStream() throws IOException {
276     if (!connected)
277         connect();
278
279     return socket.getOutputStream();
280     }
281
282     /**
283      * get the request method
284      *
285      * @return - the request method
286      *
287      */

288     public String JavaDoc getRequestMethod() {
289     return requestMethod;
290     }
291
292     /**
293      * set the request method
294      *
295      * @param String GET or HEAD
296      *
297      */

298     public void setRequestMethod(String JavaDoc value) {
299     requestMethod = value;
300     }
301
302     /**
303      * Override the default method provided in the HttpURLConnection class
304      * Get the response code for this connection
305      *
306      * @return - the response code of the connection, -1 if no valid response
307      *
308      */

309     public int getResponseCode() {
310     return responseCode;
311     }
312
313     /**
314      * Override the default method provided in the HttpURLConnection class
315      * Get the response message for the last request
316      *
317      * @return - the response message of the last request
318      *
319      */

320     public String JavaDoc getResponseMessage() {
321     return responseMessage;
322     }
323
324     /**
325      * send the request to the server
326      *
327      * @throws IOException
328      *
329      */

330     public void sendRequest() throws IOException {
331         ArrayList headers = pagedata.getHeaders();
332     //System.out.println("sendRequest() useproxy = " + useproxy + "\n" +
333
// getRequestMethod() + " " + url.toString() + " HTTP/1.1\r\n");
334
// send request to the server via http check for not ssl because if a proxy
335
// and ssl are in use the request is to be tunneled through the proxy
336
// connection and is already setup
337
if (useproxy && !pagedata.getSSL()) {
338         send(out, getRequestMethod() + " " + url.toString() + " HTTP/1.1\r\n");
339         if (pagedata.getAuth() != null)
340         send(out, "Proxy-Authorization: " + pagedata.getAuth() + "\r\n");
341             if (headers != null) {
342                 ArrayList name = (ArrayList) headers.get(0);
343                 ArrayList value = (ArrayList) headers.get(1);
344                 for (int i = 0; i < name.size(); i++) {
345                     send(out, name.get(i) + ": " + value.get(i) + "\r\n");
346         }
347         }
348         send(out, "Connection: close\r\n");
349         } else {
350         send(out, getRequestMethod() + " " + url.getFile() + " HTTP/1.1\r\n");
351         send(out, "Host: " + url.getHost() + "\r\n");
352             if (headers != null) {
353                 ArrayList name = (ArrayList) headers.get(0);
354                 ArrayList value = (ArrayList) headers.get(1);
355                 for (int i = 0; i < name.size(); i++) {
356                     send(out, name.get(i) + ": " + value.get(i) + "\r\n");
357         }
358         }
359     }
360     //send(sslout, "Accept: text/plain, text/html \r\n");
361
send(out, "\r\n");
362
363     // get the response message to the request
364
responseMessage = recv(socket.getInputStream());
365     try {
366         responseCode = Integer.parseInt(responseMessage.substring(9, 12));
367     } catch (NumberFormatException JavaDoc nfe) {responseCode = -1;}
368
369     // check for and take care of redirection
370
if (responseCode == 301 || responseCode == 302)
371         doRedirection();
372     }
373
374     /**
375      * take care of redirection if the response code is 301 or 302
376      *
377      */

378     private void doRedirection() throws IOException {
379     setURL(getHeaderField("Location"));
380     // log the redirection
381
pageContext.getServletContext().
382            log(getResponseMessage() + ": Connection redirected to " +
383                getHeaderField("Location"));
384     // close connection so that a new one can be opened to Location:
385
disconnect();
386     connect();
387     sendRequest();
388     }
389
390
391     /**
392      * make a regular http connection
393      *
394      * @throws IOException
395      *
396      */

397     boolean makeTunnelConnection() throws IOException {
398     //System.out.println("makeTunnelConnection() sending CONNECT to proxy CONNECT "
399
// + url.getHost() + ":" + port + " HTTP/1.1");
400

401     // create socket to the proxy server
402
InetAddress proxy = InetAddress.getByName(pagedata.getProxyServer());
403     // make the connection to the proxy server
404
socket = new Socket(proxy, pagedata.getProxyPort());
405
406     // get the outputstream
407
out = socket.getOutputStream();
408
409     send(out, "CONNECT " + url.getHost() + ":" + port + " HTTP/1.0\r\n");
410     if (pagedata.getAuth() != null) {
411       //System.out.println("sending the auth line Proxy-Authorization: " + auth);
412
send(out, "Proxy-Authorization: " + pagedata.getAuth() + "\r\n");
413     }
414     send(out, "\r\n");
415
416     // get the response from the proxy server for the connection
417
String JavaDoc sb = recv(socket.getInputStream());
418
419     if (sb.startsWith("HTTP/1.0 200"))
420         // connection successful
421
return true;
422         else {
423         // connection failed
424
proxyerror = sb.substring(9);
425         return false;
426     }
427     }
428
429     /**
430      * get the sslsocket
431      *
432      * @return - the created sslsocket
433      *
434      * @throws IOException
435      *
436      *
437     private void makeSecureConnection() throws IOException {
438     System.out.println("makeSecureConnection()");
439     if (useproxy) {
440         // going to tunnel the ssl connection through the proxy
441         if (!makeTunnelConnection()) {
442         // throw exception for some reason
443             throw new IOException("Connection to the proxy server could not be "
444                       + "extablished: " + proxyerror);
445         }
446     }
447     SSLSocket sslsocket = null;
448     try {
449         SSLSocketFactory sslfactory;
450         SSLContext context = SSLContext.getInstance("TLS");
451         // if client authorization is required for the ssl connection init the
452         // SSLContext with the clients keystore
453         if (sslpass != null) {
454         System.out.println("using client authentication for ssl");
455         KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
456             FileInputStream filein =
457          new FileInputStream(System.getProperty("javax.net.ssl.trustStore"));
458         KeyStore ks = KeyStore.getInstance("JKS");
459         ks.load((InputStream)filein, null);
460         System.out.println("password = " + sslpass);
461         char[] password = pagedata.getSslPass.toCharArray();
462         kmf.init(ks, password);
463         context.init(kmf.getKeyManagers(), null, null);
464                 sslfactory = context.getSocketFactory();
465         } else
466         // use the default SSLSocketFactory
467         sslfactory = (SSLSocketFactory) SSLSocketFactory.getDefault();
468
469         if (!useproxy)
470         sslsocket = (SSLSocket) sslfactory.createSocket(dst, port);
471             else {
472                 sslsocket =
473                    (SSLSocket) sslfactory.createSocket(socket, url.getHost(),
474                                port, true);
475         //System.out.println("created the tunneled socket");
476         }
477     } catch (java.security.GeneralSecurityException nsae) {
478         // should never happen
479         throw new IOException("HttpConnection.connect(): " +
480                 "creating tunneled sslsocket " + nsae.toString());
481     }
482     System.out.println("starting sslhandshake");
483     sslsocket.startHandshake();
484     socket = sslsocket;
485     out = sslsocket.getOutputStream();
486     System.out.println("makeSecureConnection() DONE");
487     }
488     */

489     /**
490      * Override default provided in URLConnection
491      *
492      * @param n - index of the desired header field
493      *
494      */

495     public String JavaDoc getHeaderField(int n) {
496         getHeaders(); // get headers from http response
497

498         if (n < headers.size()) { // make sure n is not out of bounds
499
return getField((String JavaDoc)headers.elementAt(n)); // return header field
500
}
501         return null;
502     }
503
504     /**
505      * Override default provided in URLConnection
506      *
507      * @return - LastModified header or 0 if LastModified does not exist
508      *
509      */

510     public long getLastModified() {
511         getHeaders(); // get headers from http response
512

513         String JavaDoc header = (String JavaDoc)keys.get("LastModified");
514         if (header != null) {
515             try {
516             return new Integer JavaDoc(header).longValue();
517             } catch (NumberFormatException JavaDoc nfe) {
518             return 0;
519             }
520         }
521     return 0;
522     }
523
524     /**
525      * Override default provided in URLConnection
526      *
527      * @return - the Expiration header or 0 if Expiration does not exist
528      *
529      */

530     public long getExpiration() {
531         getHeaders(); // get headers from http response
532

533     String JavaDoc header = (String JavaDoc)keys.get("Expiration");
534         if (header != null) {
535             try {
536             return new Integer JavaDoc(header).longValue();
537             } catch (NumberFormatException JavaDoc nfe) {
538             return 0;
539             }
540         }
541     return 0;
542     }
543
544     /**
545      * Override default provided in URLConnection
546      *
547      * @param key - name of the desired header field
548      *
549      */

550     public String JavaDoc getHeaderField(String JavaDoc key) {
551         getHeaders(); // get headers from http response
552

553         return (String JavaDoc)keys.get(key.toLowerCase()); // return desired header field
554
}
555
556     /**
557      * Override default provided in URLConnection
558      *
559      * @param n value returned if header is null
560      * @param key header to retreive
561      *
562      * @return - header as an integer
563      */

564     public int getHeaderFieldInt(String JavaDoc key, int n) {
565         getHeaders(); // get headers from http response
566

567     String JavaDoc header = (String JavaDoc)keys.get(key.toLowerCase());
568         if (header != null) {
569         try {
570         n = new Integer JavaDoc(header).intValue();
571         } catch (NumberFormatException JavaDoc nfe) {
572         // do nothing the default value specified will be returned
573
}
574     }
575     return n;
576     }
577
578     /**
579      * Override default provided in URLConnection
580      *
581      * @param n - index of the desired header field
582      *
583      */

584     public String JavaDoc getHeaderFieldKey(int n) {
585         getHeaders(); // get headers from http response
586

587         if (n < headers.size()) { // make sure n is not out of bounds
588
// return name of nth header field
589
return getKey((String JavaDoc)headers.elementAt(n));
590         }
591         return null;
592     }
593
594     /**
595      * Set the URL for this connection
596      *
597      * @throws MalformedURLException
598      *
599      */

600     protected void setURL(String JavaDoc value) throws MalformedURLException {
601     url = new URL(value);
602     }
603
604     /**
605      * get the URL of this connection
606      *
607      * @return - the URL for this connection
608      */

609     public URL getURL() {
610     return url;
611     }
612
613     /**
614      * Helper routine for parsing header field
615      *
616      * @param str - complete header field name and value separated by :
617      *
618      * @return - name of header field
619      *
620      */

621     private String JavaDoc getKey(String JavaDoc str) {
622         if (str == null)
623             return null;
624         int ind = str.indexOf(keySeparator);
625         if (ind >= 0)
626             return str.substring(0, ind).toLowerCase();
627         return null;
628     }
629
630     /**
631      * Helper routine for parsing header field
632      *
633      * @param str - complete header field name and value separated by :
634      *
635      * @return value of header field
636      *
637      */

638     private String JavaDoc getField(String JavaDoc str) {
639         if (str == null)
640             return null;
641         int ind = str.indexOf(keySeparator);
642         if (ind >= 0)
643             return str.substring(ind+1).trim();
644         else
645             return str;
646     }
647
648     /**
649      * Helper routine that reads header from HTTP connection
650      *
651      */

652     private void getHeaders() {
653
654     String JavaDoc header; // holds one full header name value pair
655

656     // check if headers have been gotten previously
657
if (gotten)
658             return;
659         gotten = true; // set gotten
660

661     // initialize keys
662
keys = new HashMap();
663
664         try {
665             connect(); // make connection
666

667         // get inputstream from connection
668
InputStream in = getInputStream();
669
670         header = recv(in); // get initial header field
671

672             while (true) {
673                 if (header.length() == 0) // exit loop if no more headers to parse
674
break;
675         headers.addElement(header); // build the vector headers
676
String JavaDoc key = getKey(header); // parse the header name
677
if (key != null) {
678                     keys.put(key, getField(header)); // parse the header value
679
}
680                 header = recv(in); // get next header
681
}
682         } catch (IOException e) {
683             e.printStackTrace();
684         }
685     }
686
687     /**
688      * helper routine to send output stream over the http connection
689      *
690      * @param msg - String sent via out to http server
691      * @param out - OutputStream msg is written to
692      *
693      * @throws IOException when a problem occurs with the write
694      */

695     private void send(OutputStream out, String JavaDoc msg) throws IOException {
696     // create outputstreamwriter to write whole String
697
OutputStreamWriter output = new OutputStreamWriter(out);
698
699     output.write(msg, 0, msg.length()); // write to output stream
700
output.flush();
701     }
702
703     /**
704      * Helper routine to read a newline-terminated string from input stream
705      *
706      * @param in - inputstream to read from
707      *
708      * @throws IOException when a problem occurs with the read
709      *
710      */

711     private String JavaDoc recv(InputStream in) throws IOException {
712         String JavaDoc result = ""; // final result from read to be returned
713
int c = in.read(); // read in initial char from inputstream
714

715         while (c >= 0 && c != '\n') {
716             if (c != '\r') {
717                 result += (char)c;
718             }
719             c = in.read();
720         }
721     //System.out.println("The results of the read = " + result);
722
return result;
723     }
724 }
725
Popular Tags