Java > Open Source Codes > org > openlaszlo > data > HTTPDataSource


1 /* *****************************************************************************
2  * HTTPDataSource.java
3  * ****************************************************************************/
4
5 /* J_LZ_COPYRIGHT_BEGIN *******************************************************
6 * Copyright 2001-2004 Laszlo Systems, Inc. All Rights Reserved. *
7 * Use is subject to license terms. *
8 * J_LZ_COPYRIGHT_END *********************************************************/
9
10 package org.openlaszlo.data;
11
12 import java.util.Enumeration ;
13 import java.util.Hashtable ;
14 import java.util.StringTokenizer ;
15 import java.net.URL ;
16 import java.net.URLDecoder ;
17 import java.net.MalformedURLException ;
18 import java.net.UnknownHostException ;
19 import java.io.*;
20 import javax.servlet.http.HttpServletRequest ;
21 import javax.servlet.http.HttpServletResponse ;
22
23 import org.apache.commons.httpclient.*;
24 import org.apache.commons.httpclient.methods.*;
25 import org.apache.commons.httpclient.util.*;
26 import org.apache.log4j.*;
27
28 import org.openlaszlo.xml.internal.XMLUtils;
29 import org.openlaszlo.utils.LZHttpUtils;
30 import org.openlaszlo.utils.LZGetMethod;
31 import org.openlaszlo.utils.LZPostMethod;
32 import org.openlaszlo.utils.FileUtils;
33 import org.openlaszlo.server.LPS;
34 import org.apache.oro.text.regex.*;
35
36
37 /**
38  * HTTP Transport
39  */
40 public class HTTPDataSource extends DataSource {
41     
42     private static Logger mLogger = Logger.getLogger(HTTPDataSource.class);
43
44     /** Connection Manager */
45     private static MultiThreadedHttpConnectionManager
46         mConnectionMgr = null;
47
48     /** max number of http retries */
49     private static int mMaxRetries = 1;
50
51     /** Whether or not to use the http11 . */
52     private static boolean mUseHttp11 = true;
53
54     /** Connection timeout millis (0 means default) */
55     private static int mConnectionTimeout = 0;
56
57     /** Timeout millis (0 means infinity) */
58     private static int mTimeout = 0;
59
60     /** Connection pool timeout in millis (0 means infinity) */
61     private static int mConnectionPoolTimeout = 0;
62
63     /** Max total connections. */
64     private static int mMaxTotalConnections = 1000;
65
66     /** Max connections per host. */
67     private static int mMaxConnectionsPerHost = mMaxTotalConnections;
68
69     /** Number of backend redirects we allow (potential security hole) */
70     private static int mFollowRedirects = 0;
71
72     {
73         String useMultiThreadedConnectionMgr = LPS.getProperty("http.useConnectionPool", "true");
74
75         if (Boolean.valueOf(useMultiThreadedConnectionMgr).booleanValue()) {
76             mLogger.info("using connection pool");
77             mConnectionMgr = new MultiThreadedHttpConnectionManager();
78         } else {
79             mLogger.info("not using connection pool");
80         }
81     
82         // Parse multi connection properties anyway. May be used by AXIS. See
83 // ResponderCache for details.
84 {
85             String maxConns = LPS.getProperty("http.maxConns", "1000");
86             mMaxTotalConnections = Integer.parseInt(maxConns);
87             if (mConnectionMgr != null) {
88                 mConnectionMgr.setMaxTotalConnections(mMaxTotalConnections);
89             }
90     
91             maxConns = LPS.getProperty("http.maxConnsPerHost", maxConns);
92             mMaxConnectionsPerHost = Integer.parseInt(maxConns);
93             if (mConnectionMgr != null) {
94                 mConnectionMgr.setMaxConnectionsPerHost(mMaxConnectionsPerHost);
95             }
96         }
97
98         String maxRetries = LPS.getProperty("http.maxBackendRetries", "1");
99         mMaxRetries = Integer.parseInt(maxRetries);
100
101         String followRedirects = LPS.getProperty("http.followRedirects", "0");
102         mFollowRedirects = Integer.parseInt(followRedirects);
103
104         String timeout = LPS.getProperty("http.backendTimeout", "30000");
105         mTimeout = Integer.parseInt(timeout);
106
107         timeout = LPS.getProperty("http.backendConnectionTimeout", timeout);
108         mConnectionTimeout = Integer.parseInt(timeout);
109
110         timeout = LPS.getProperty("http.connectionPoolTimeout", "0");
111         mConnectionPoolTimeout = Integer.parseInt(timeout);
112
113         String useHttp11 = LPS.getProperty("http.useHttp11", "true");
114         mUseHttp11 = Boolean.valueOf(useHttp11).booleanValue();
115         if (mUseHttp11) {
116             mLogger.info("using HTTP 1.1");
117         } else {
118             mLogger.info("not using HTTP 1.1");
119         }
120     }
121
122
123     /**
124      * @return name of this datasource
125      */
126     public String name() {
127         return "http";
128     }
129
130     /**
131      * Do an HTTP Get/Post based on this request
132      *
133      * @return the data from this request
134      * @param app absolute pathnane to app file
135      * @param req request in progress (possibly null)
136      * @param since this is the timestamp on the
137      * currently cached item; this time can be used as the datasource
138      * sees fit (or ignored) in constructing the results. If
139      * the value is -1, assume there is no currently cached item.
140      */
141     public Data getData(String app, HttpServletRequest req, HttpServletResponse res, long since)
142         throws DataSourceException, IOException {
143         return getHTTPData(req, res, getURL(req), since);
144     }
145
146     public static Data getHTTPData(HttpServletRequest req, HttpServletResponse res,
147                                    String surl, long since)
148         throws DataSourceException, IOException {
149
150         int tries = 1;
151
152         // timeout msecs of time we're allowed in this routine
153 // we must return or throw an exception. 0 means infinite.
154 int timeout = mTimeout;
155         if (req != null) {
156             String timeoutParm = req.getParameter("timeout");
157             if (timeoutParm != null) {
158                 timeout = Integer.parseInt(timeoutParm);
159             }
160         }
161
162         long t1 = System.currentTimeMillis();
163         long elapsed = 0;
164         if (surl == null) {
165             surl = getURL(req);
166         }
167
168         while(true) {
169             String m = null;
170
171             long tout;
172             if (timeout > 0) {
173                 tout = timeout - elapsed;
174                 if (tout <= 0) {
175                     throw new InterruptedIOException(surl + " timed out");
176                 }
177             } else {
178                 tout = 0;
179             }
180
181             try {
182                 HttpData data = getDataOnce(req, res, since, surl, 0, (int)tout);
183                 if (data.code >= 400) {
184                     data.release();
185                     throw new DataSourceException(errorMessage(data.code));
186                 }
187                 return data;
188             } catch (HttpRecoverableException e) {
189                 // This type of exception should be retried.
190 if (tries++ > mMaxRetries) {
191                     throw new InterruptedIOException("too many retries, exception: " + e.getMessage());
192                 }
193                 mLogger.warn("retrying a recoverable exception: " + e.getMessage());
194             } catch (HttpException e) {
195                 String msg = "HttpException: " + e.getMessage();
196                 throw new IOException("HttpException: " + e.getMessage());
197             } catch (IOException e) {
198
199                 try {
200                     Class ssle = Class.forName("javax.net.ssl.SSLException");
201                     if (ssle.isAssignableFrom(e.getClass())) {
202                         throw new DataSourceException("SSL exception: " +
203                                 e.getMessage());
204                     }
205                 } catch (ClassNotFoundException cfne) {
206                 }
207
208                 throw e;
209             }
210
211             long t2 = System.currentTimeMillis();
212             elapsed = (t2 - t1);
213         }
214     }
215
216     /**
217      * convenience routine missing from http library
218      */
219     static boolean isRedirect(int rc) {
220         return (rc == HttpStatus.SC_MOVED_PERMANENTLY ||
221                 rc == HttpStatus.SC_MOVED_TEMPORARILY ||
222                 rc == HttpStatus.SC_SEE_OTHER ||
223                 rc == HttpStatus.SC_TEMPORARY_REDIRECT);
224     }
225     /**
226      * @param since last modified time to use
227      * @param req
228      * @param url if null, ignored
229      * @param redirCount number of redirs we've done
230      */
231     public static HttpData getDataOnce(HttpServletRequest req,
232          HttpServletResponse res, long since, String surl,
233          int redirCount, int timeout)
234          throws IOException, HttpException, DataSourceException, MalformedURLException {
235
236         GetMethod request = null;
237         HostConfiguration hcfg = new HostConfiguration();
238
239         if (res != null) {
240             res.setContentType("application/x-www-form-urlencoded;charset=UTF-8");
241         }
242
243         try {
244
245             // TODO: [2002-01-09 bloch] cope with cache-control
246 // response headers (no-store, no-cache, must-revalidate,
247 // proxy-revalidate).
248
249             if (surl == null) {
250                 surl = getURL(req);
251             }
252             if (surl == null || surl.equals("")) {
253                 throw new MalformedURLException ("url is empty or null");
254             }
255     
256             String reqType = "";
257             String headers = "";
258
259             if (req != null) {
260                 reqType = req.getParameter("reqtype");
261                 headers = req.getParameter("headers");
262             }
263     
264             boolean isPost = false;
265     
266             if (reqType != null && reqType.equals("POST")) {
267                 request = new LZPostMethod();
268                 request.setRequestHeader("Content-Type",
269                                          "application/x-www-form-urlencoded;charset=UTF-8");
270                 isPost = true;
271             } else {
272                 request = new LZGetMethod();
273             }
274
275
276             request.setHttp11(mUseHttp11);
277
278             // Proxy the request headers
279 if (req != null) {
280                 LZHttpUtils.proxyRequestHeaders(req, request);
281             }
282     
283             // Set headers from query string
284 if (headers != null && headers.length() > 0) {
285                 StringTokenizer st = new StringTokenizer (headers, "\n");
286                 while (st.hasMoreTokens()) {
287                     String h = st.nextToken();
288                     int i = h.indexOf(":");
289                     if (i > -1) {
290                         String n = h.substring(0, i);
291                         String v = h.substring(i + 2, h.length());
292                         request.setRequestHeader( n , v );
293                         mLogger.debug(" setting header " + n + "=" + v);
294                     }
295                 }
296             }
297     
298             mLogger.debug("Parsing url");
299             URI uri = LZHttpUtils.newURI(surl);
300             try {
301                 hcfg.setHost(uri);
302             } catch (Exception e) {
303                 throw new MalformedURLException ("can't form uri from " + surl);
304             }
305     
306             // This gets us the url-encoded (escaped) path and query string
307 String path = uri.getEscapedPath();
308             String query = uri.getEscapedQuery();
309             mLogger.debug("encoded path: " + path);
310             mLogger.debug("encoded query: " + query);
311     
312             // This call takes a decoded (unescaped) path
313 request.setPath(path);
314     
315             boolean hasQuery = (query != null && query.length() > 0);
316     
317             if (isPost) {
318                 if (hasQuery) {
319                     final String postbodyparam = "lzpostbody=";
320                     if (query.startsWith(postbodyparam)) {
321                         // Get the unescaped query string
322 String v = uri.getQuery().substring(postbodyparam.length());
323                         ((LZPostMethod)request).setRequestBody(v);
324                     } else {
325                         StringTokenizer st = new StringTokenizer (query, "&");
326                         while (st.hasMoreTokens()) {
327                             String it = st.nextToken();
328                             int i = it.indexOf("=");
329                             if (i > 0) {
330                                 String n = it.substring(0, i);
331                                 String v = it.substring(i + 1, it.length());
332                                 // POST encodes values during request
333 ((PostMethod)request).addParameter(n, URLDecoder.decode(v, "UTF-8"));
334                             } else {
335                                 mLogger.warn("ignoring bad token (missing '=' char) in query string: " + it);
336                             }
337                         }
338                     }
339                 }
340             } else {
341                 // This call takes an encoded (escaped) query string
342 request.setQueryString(query);
343             }
344     
345             // Put in the If-Modified-Since headers
346 if (since != -1) {
347                 String lms = LZHttpUtils.getDateString(since);
348                 request.setRequestHeader(LZHttpUtils.IF_MODIFIED_SINCE, lms);
349                 mLogger.debug("proxying lms: " + lms);
350             }
351                 
352             mLogger.debug("setting up http client");
353             HttpClient htc = null;
354             if (mConnectionMgr != null) {
355                 htc = new HttpClient(mConnectionMgr);
356             } else {
357                 htc = new HttpClient();
358             }
359
360             htc.setHostConfiguration(hcfg);
361     
362             // This is the data timeout
363 mLogger.debug("timeout set to " + timeout);
364             htc.setTimeout(timeout);
365     
366             // Set connection timeout the same
367 htc.setConnectionTimeout(mConnectionTimeout);
368     
369             // Set timeout for getting a connection
370 htc.setHttpConnectionFactoryTimeout(mConnectionPoolTimeout);
371     
372             // TODO: [2003-03-05 bloch] this should be more configurable (per app?)
373 request.setFollowRedirects(mFollowRedirects > 0);
374     
375             long t1 = System.currentTimeMillis();
376             mLogger.debug("starting remote request");
377             int rc = htc.executeMethod(hcfg, request);
378             String status = HttpStatus.getStatusText(rc);
379             if (status == null) {
380                 status = "" + rc;
381             }
382             mLogger.debug("remote response status: " + status);
383     
384             HttpData data = null;
385             if ( isRedirect(rc) && mFollowRedirects > redirCount ) {
386                 String loc = request.getResponseHeader("Location").toString();
387                 String hostURI = loc.substring(loc.indexOf(": ") + 2, loc.length() ) ;
388                 mLogger.info("Following URL from redirect: " + hostURI);
389                 long t2 = System.currentTimeMillis();
390                 if (timeout > 0) {
391                     timeout -= (t2 - t1);
392                     if (timeout < 0) {
393                         throw new InterruptedIOException(surl + " timed out after redirecting to " + loc);
394                     }
395                 }
396
397                 data = getDataOnce(req, res, since, hostURI, redirCount++, timeout);
398             } else {
399                 data = new HttpData(request, rc);
400             }
401
402             if (req != null && res != null) {
403                 // proxy response headers
404 LZHttpUtils.proxyResponseHeaders(request, res, req.isSecure());
405             }
406
407             return data;
408
409         } catch (HttpConnection.ConnectionTimeoutException ce) {
410             // Transduce to an InterrupedIOException, since lps takes these to be timeouts.
411 if (request != null) {
412                 request.releaseConnection();
413             }
414             throw new InterruptedIOException("connecting to " + hcfg.getHost() + ":" + hcfg.getPort() +
415                         " timed out beyond " + mConnectionTimeout + " msecs.");
416         } catch (HttpRecoverableException hre) {
417             if (request != null) {
418                 request.releaseConnection();
419             }
420             throw hre;
421         } catch (HttpException e) {
422             if (request != null) {
423                 request.releaseConnection();
424             }
425             throw e;
426         } catch (IOException ie) {
427             if (request != null) {
428                 request.releaseConnection();
429             }
430             throw ie;
431         } catch (RuntimeException e) {
432            if (request != null) {
433                request.releaseConnection();
434            }
435            throw e;
436         }
437     }
438     
439     /**
440      * utility
441      */
442     private static String errorMessage(int code) {
443         return "HTTP Status code: " + code + ":" +
444                 HttpStatus.getStatusText(code);
445     }
446
447     /**
448      * A class for holding on to results of an Http fetch.
449      *
450      * @author <a HREF="mailto:bloch@laszlosystems.com">Eric Bloch</a>
451      */
452
453     public static class HttpData extends Data {
454
455         /** response code */
456         public final int code;
457     
458         /** Http request */
459         public final GetMethod request;
460     
461         private PatternMatcher pMatcher = new Perl5Matcher();
462         private static final Pattern charsetPattern;
463         private static final Pattern declEncodingPattern;
464         static {
465             try {
466                 Perl5Compiler compiler = new Perl5Compiler();
467                 charsetPattern = compiler.compile(";charset=([^ ]*)");
468                 declEncodingPattern =
469                     compiler.compile("[ \t\r\n]*<[?]xml .*encoding=[\"']([^ \"']*)[\"'] .*[?]>");
470             } catch (MalformedPatternException e) {
471                 throw new RuntimeException (e.getMessage());
472             }
473         }
474
475         /**
476          * @param r filled request
477          * @param c response code
478          */
479         public HttpData(GetMethod r, int c) {
480             code = c;
481             request = r;
482         }
483     
484         /**
485          * @return true if the data was "not modified"
486          */
487         public boolean notModified() {
488             return code == HttpServletResponse.SC_NOT_MODIFIED;
489         }
490     
491         /**
492          * @return the lastModified time of the data
493          */
494         public long lastModified() {
495     
496             Header lastModifiedHdr = request.getResponseHeader(
497                 LZHttpUtils.LAST_MODIFIED);
498                         
499             if (lastModifiedHdr != null) {
500                 String lm = lastModifiedHdr.getValue();
501                 mLogger.debug("data with last modified at " + lm);
502                 long l = LZHttpUtils.getDate(lm);
503                 // Truncate to nearest second
504 return ((l)/1000L) * 1000L;
505             } else {
506                 mLogger.debug("data has no mod time");
507                 return -1;
508             }
509         }
510     
511         /**
512          * append response headers
513          */
514         public void appendResponseHeadersAsXML(StringBuffer xmlResponse) {
515     
516             Header[] hedz = request.getResponseHeaders();
517             for (int i = 0; i < hedz.length; i++) {
518                 String name = hedz[i].getName();
519                 if (LZHttpUtils.allowForward(name, null)) {
520                     xmlResponse.append("<header name=\""+ XMLUtils.escapeXml( name ) + "\" "
521                                      + "value=\"" + XMLUtils.escapeXml( hedz[i].getValue() ) + "\" />");
522                 }
523             }
524         }
525     
526         /**
527          * release any resources associated with this data
528          */
529         public void release() {
530             request.releaseConnection();
531         }
532
533         /**
534          * @return mime type
535          */
536         public String getMimeType() {
537             Header hdr = request.getResponseHeader(LZHttpUtils.CONTENT_TYPE);
538             String contentType = "";
539             if (hdr != null) {
540                 contentType = hdr.getValue();
541             }
542             mLogger.debug("content type: " + contentType);
543             return contentType;
544         }
545
546         /**
547          * @return string
548          */
549         public String getAsString() throws IOException {
550             byte rawbytes[] = request.getResponseBody();
551             if (rawbytes == null || rawbytes.length == 0) {
552                 throw new InterruptedIOException("null http response body");
553             }
554             String encoding = "UTF-8";
555             String content = getMimeType();
556             // search for ;charset=XXXX in Content-Type header
557 if (pMatcher.matches(content, charsetPattern)) {
558                 encoding = pMatcher.getMatch().group(1);
559             }
560             // search for 'encoding' attribute in xml declaration, e.g.,
561 // <?xml version="1.0" encoding="ISO-8859-1" standalone="no"?>
562
563             String decl = getXMLDeclaration(rawbytes);
564             if (pMatcher.matches(decl, declEncodingPattern)) {
565                 encoding = pMatcher.getMatch().group(1);
566                 //mLogger.debug("parsed data encoding: " + encoding);
567 }
568             
569             return new String (rawbytes, encoding);
570         }
571
572         /** Returns the first non-whitespace line.
573          *
574          */
575         String getXMLDeclaration(byte buf[]) {
576             String str = new String (buf);
577             BufferedReader br = new BufferedReader(new StringReader(str));
578             String line;
579             while (true) {
580                 try { line = br.readLine(); } catch (IOException e) { return ""; }
581                 if (line == null) {
582                     return "";
583                 }
584                 if (line.length() == 0) continue;
585                 if (line.startsWith("<?xml ")) {
586                     return line;
587                 } else {
588                     return "";
589                 }
590             }
591         }
592
593         /**
594          * @return input stream
595          */
596         public InputStream getInputStream() throws IOException {
597             InputStream str = request.getResponseBodyAsStream();
598             if (str == null) {
599                 throw new IOException("http response body is null");
600             }
601             return str;
602         }
603
604         /**
605          * @return size, if known
606          */
607         public long size() {
608             Header hdr = request.getResponseHeader(LZHttpUtils.CONTENT_LENGTH);
609             if (hdr != null) {
610                 String contentLength = hdr.getValue();
611                 if (contentLength != null) {
612                     mLogger.debug("content length: " + contentLength);
613                     int cl = Integer.parseInt(contentLength);
614                     return cl;
615                 }
616             }
617             return -1;
618         }
619     }
620
621     public static int getConnectionPoolTimeout() {
622         return mConnectionPoolTimeout;
623     }
624
625     public static int getMaxTotalConnections() {
626         return mMaxTotalConnections;
627     }
628
629     public static int getMaxConnectionsPerHost() {
630         return mMaxConnectionsPerHost;
631     }
632
633     public static void main(String args[]) {
634
635         HTTPDataSource ds = new HTTPDataSource();
636
637         try {
638             if (args.length != 1 && args.length != 2) {
639                 throw new Exception ("Need an url");
640             }
641             String surl = args[0];
642             FileOutputStream out = null;
643             if (args.length == 2) {
644                 out = new FileOutputStream(args[1]);
645             }
646             System.out.println("url is " + surl);
647
648             HttpData data = ds.getDataOnce(null, null, -1, surl, 0, 0);
649
650             System.out.println("Response code: " + data.code);
651
652             if (out != null) {
653                 FileUtils.send(data.getInputStream(), out);
654             }
655
656             data.release();
657             
658         } catch (Exception e) {
659             e.printStackTrace();
660         }
661     }
662 }
663

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags