ConnectionPool


1   /*
2    * This software is released under a licence similar to the Apache Software Licence.
3    * See org.logicalcobwebs.proxool.package.html for details.
4    * The latest version is available at http://proxool.sourceforge.net
5    */
6   package org.logicalcobwebs.proxool;
7   
8   import java.sql.Connection  ;
9   import java.sql.SQLException  ;
10  import java.util.Collection  ;
11  import java.util.Date  ;
12  import java.util.HashSet  ;
13  import java.util.Iterator  ;
14  import java.util.List  ;
15  import java.util.Set  ;
16  import java.util.TreeSet  ;
17  
18  import org.logicalcobwebs.concurrent.ReaderPreferenceReadWriteLock;
19  import org.logicalcobwebs.concurrent.WriterPreferenceReadWriteLock;
20  import org.apache.commons.logging.Log;
21  import org.apache.commons.logging.LogFactory;
22  import org.logicalcobwebs.proxool.admin.Admin;
23  import org.logicalcobwebs.proxool.util.FastArrayList;
24  
25  /**
26   * This is where most things happen. (In fact, probably too many things happen in this one
27   * class).
28   * @version $Revision: 1.84 $, $Date: 2006/03/23 11:44:57 $
29   * @author billhorsman
30   * @author $Author: billhorsman $ (current maintainer)
31   */
32  class ConnectionPool implements ConnectionPoolStatisticsIF {
33  
34      /**
35       * Use this for messages that aren't useful for the pool specific log
36       */
37      private static final Log LOG = LogFactory.getLog(ConnectionPool.class);
38  
39      /**
40       * Here we deviate from the standard of using the classname for the log
41       * name. Here we want to use the alias for the pool so that we can log
42       * each pool to different places. So we have to instantiate the log later.
43       */
44      private Log log;
45  
46      private ReaderPreferenceReadWriteLock connectionStatusReadWriteLock = new ReaderPreferenceReadWriteLock();
47  
48      /**
49       * If you want to shutdown the pool you should get a write lock on this. And if you use the pool then
50       * get a read lock. This stops us trying to shutdown the pool whilst it is in use. Only, we don't want
51       * to delay shutdown just because some greedy user has got a connection active. Shutdown should be
52       * relatively immediate. So we don't ask for a read lock for the whole time that a connection is active.
53       */
54      private WriterPreferenceReadWriteLock primaryReadWriteLock = new WriterPreferenceReadWriteLock();
55  
56      private static final String  [] STATUS_DESCRIPTIONS = {"NULL", "AVAILABLE", "ACTIVE", "OFFLINE"};
57  
58      private static final String   MSG_MAX_CONNECTION_COUNT =
59              "Couldn't get connection because we are at maximum connection count and there are none available";
60  
61      /** This is the pool itself */
62      private List   proxyConnections;
63  
64      /** This is the "round robin" that makes sure we use all the connections */
65      private int nextAvailableConnection = 0;
66  
67      private long connectionsServedCount = 0;
68  
69      private long connectionsRefusedCount = 0;
70  
71      /** This keeps a count of how many connections there are in each state */
72      private int[] connectionCountByState = new int[4];
73  
74      private ConnectionPoolDefinition definition;
75  
76      private CompositeConnectionListener compositeConnectionListener = new CompositeConnectionListener();
77  
78      private CompositeStateListener compositeStateListener = new CompositeStateListener();
79  
80      private long timeOfLastRefusal = 0;
81  
82      private int upState;
83  
84      private static boolean loggedLegend;
85  
86      private Admin admin;
87  
88      private boolean locked = false;
89  
90      private Date   dateStarted = new Date  ();
91  
92      private boolean connectionPoolUp = false;
93  
94      /**
95       * This gets set during {@link #shutdown}. We use it to notify shutdown
96       * that all connections are now non-active.
97       */
98      private Thread   shutdownThread;
99  
100     private Prototyper prototyper;
101 
102     /**
103      * Initialised in {@link ConnectionPool#ConnectionPool constructor}.
104      */
105     private ConnectionResetter connectionResetter;
106 
107     private ConnectionValidatorIF connectionValidator;
108     
109     
110     protected ConnectionPool(ConnectionPoolDefinition definition) throws ProxoolException {
111 
112         // Use the FastArrayList for performance and thread safe
113         // behaviour. We set its behaviour to "fast"  (meaning reads are
114         // unsynchronized, whilst writes are not).
115         FastArrayList fal = new FastArrayList();
116         fal.setFast(true);
117         proxyConnections = fal;
118 
119         log = LogFactory.getLog("org.logicalcobwebs.proxool." + definition.getAlias());
120         connectionResetter = new ConnectionResetter(log, definition.getDriver());
121         setDefinition(definition);
122 
123         connectionValidator = new DefaultConnectionValidator();
124         
125         if (definition.getStatistics() != null) {
126             try {
127                 admin = new Admin(definition);
128             } catch (ProxoolException e) {
129                 log.error("Failed to initialise statistics", e);
130             }
131         }
132 
133         ShutdownHook.init();
134     }
135 
136     /** Starts up house keeping and prototyper threads. */
137     protected void start() throws ProxoolException {
138         connectionPoolUp = true;
139         prototyper = new Prototyper(this);
140         HouseKeeperController.register(this);
141     }
142 
143     /**
144      * Get a connection from the pool.  If none are available or there was an Exception
145      * then an exception is thrown and something written to the log
146      */
147     protected Connection   getConnection() throws SQLException   {
148 
149         String   requester = Thread.currentThread().getName();
150 
151         /*
152          *If we're busy, we need to return as quickly as possible. Because this is unsynchronized
153          * we run the risk of refusing a connection when we might actually be able to. But that will
154          * only happen when we're right at or near maximum connections anyway.
155          */
156 
157         try {
158             prototyper.quickRefuse();
159         } catch (SQLException   e) {
160             connectionsRefusedCount++;
161             if (admin != null) {
162                 admin.connectionRefused();
163             }
164             log.info(displayStatistics() + " - " + MSG_MAX_CONNECTION_COUNT);
165             timeOfLastRefusal = System.currentTimeMillis();
166             setUpState(StateListenerIF.STATE_OVERLOADED);
167             throw e;
168         }
169 
170         prototyper.checkSimultaneousBuildThrottle();       
171         
172         ProxyConnection proxyConnection = null;
173 
174         try {
175 
176             // We need to look at all the connections, but we don't want to keep looping round forever
177             for (int connectionsTried = 0; connectionsTried < proxyConnections.size(); connectionsTried++) {
178                 // By doing this in a try/catch we avoid needing to synch on the size().  We need to do be
179                 // able to cope with connections being removed whilst we are going round this loop
180                 try {
181                     proxyConnection = (ProxyConnection) proxyConnections.get(nextAvailableConnection);
182                 } catch (ArrayIndexOutOfBoundsException   e) {
183                     // This is thrown by a Vector (which we no longer use), but is
184                     // kept here for a while.
185                     nextAvailableConnection = 0;
186                     proxyConnection = (ProxyConnection) proxyConnections.get(nextAvailableConnection);
187                 } catch (IndexOutOfBoundsException   e) {
188                     // This is thrown by a true List
189                     nextAvailableConnection = 0;
190                     proxyConnection = (ProxyConnection) proxyConnections.get(nextAvailableConnection);
191                 }
192                 // setActive() returns false if the ProxyConnection wasn't available.  You
193                 // can't set it active twice (at least, not without making it available again
194                 // in between)
195                 if (proxyConnection != null && proxyConnection.setStatus(ProxyConnectionIF.STATUS_AVAILABLE, ProxyConnectionIF.STATUS_ACTIVE)) {
196 
197                     // Okay. So we have it. But is it working ok?
198                     if (getDefinition().isTestBeforeUse()) {
199                         if (!testConnection(proxyConnection)) {
200                             // Oops. No it's not. Let's choose another.
201                             proxyConnection = null;
202                         }
203                     }
204                     if (proxyConnection != null) {
205                         nextAvailableConnection++;
206                         break;
207                     }
208                 } else {
209                     proxyConnection = null;
210                 }
211                 nextAvailableConnection++;
212             }
213             // Did we get one?
214             if (proxyConnection == null) {
215                 try {
216                     // No!  Let's see if we can create one
217                     proxyConnection = prototyper.buildConnection(ProxyConnection.STATUS_ACTIVE, "on demand");
218                     
219                     // Okay. So we have it. But is it working ok?
220                     if (getDefinition().isTestBeforeUse()) {
221                         if (!testConnection(proxyConnection)) {
222                             // Oops. No it's not. There's not much more we can do for now
223                             throw new SQLException  ("Created a new connection but it failed its test");
224                         }
225                     }
226                 } catch (SQLException   e) {
227                     throw e;
228                 } catch (ProxoolException e) {
229                     log.debug("Couldn't get connection", e);
230                     throw new SQLException  (e.toString());
231                 } catch (Throwable   e) {
232                     log.error("Couldn't get connection", e);
233                     throw new SQLException  (e.toString());
234                 }
235             }
236 
237         } catch (SQLException   e) {
238             throw e;
239         } catch (Throwable   t) {
240             log.error("Problem getting connection", t);
241             throw new SQLException  (t.toString());
242         } finally {
243             if (proxyConnection != null) {
244                 connectionsServedCount++;
245                 proxyConnection.setRequester(requester);
246             } else {
247                 connectionsRefusedCount++;
248                 if (admin != null) {
249                     admin.connectionRefused();
250                 }
251                 timeOfLastRefusal = System.currentTimeMillis();
252                 setUpState(StateListenerIF.STATE_OVERLOADED);
253             }
254         }
255 
256         if (proxyConnection == null) {
257             throw new SQLException  ("Unknown reason for not getting connection. Sorry.");
258         }
259 
260         if (log.isDebugEnabled() && getDefinition().isVerbose()) {
261             log.debug(displayStatistics() + " - Connection #" + proxyConnection.getId() + " served");
262         }
263 
264         // This gives the proxy connection a chance to reset itself before it is served.
265         proxyConnection.open();
266 
267         return ProxyFactory.getWrappedConnection(proxyConnection);
268     }
269 
270     /**
271      * Test the connection (if required)
272      * If the connection fails the test, it is removed from the pool.
273      * If no ConnectionValidatorIF is defined, then the test always succeed.
274      * 
275      * @param proxyConnection the connection to test
276      * @return TRUE if the connection pass the test, FALSE if it fails
277      */
278     private boolean testConnection(ProxyConnectionIF proxyConnection) {
279         // is validation enabled ?
280         if( connectionValidator == null ) {
281             return true;
282         }
283         
284         // validate the connection
285         boolean success = connectionValidator.validate(getDefinition(), proxyConnection.getConnection());
286         
287         if( success ) {
288             if (LOG.isDebugEnabled()) {
289                 LOG.debug(displayStatistics() + " - Connection #" + proxyConnection.getId() + " tested: OK");
290             }
291         }
292         else {
293             proxyConnection.setStatus(ProxyConnectionIF.STATUS_NULL);
294             removeProxyConnection(proxyConnection, "it didn't pass the validation", ConnectionPool.REQUEST_EXPIRY, true);
295         }
296         
297         // return 
298         return success;
299     }
300 
301     /**
302      * Add a ProxyConnection to the pool
303      * @param proxyConnection new connection
304      * @return true if the connection was added or false if it wasn't (for instance, if the definition it
305      * was built with is out of date).
306      */
307     protected boolean addProxyConnection(ProxyConnectionIF proxyConnection) {
308         boolean added = false;
309         try {
310             acquireConnectionStatusWriteLock();
311             if (proxyConnection.getDefinition() == getDefinition()) {
312                 proxyConnections.add(proxyConnection);
313                 connectionCountByState[proxyConnection.getStatus()]++;
314                 added = true;
315             }
316         } finally {
317             releaseConnectionStatusWriteLock();
318         }
319         return added;
320     }
321 
322     protected static String   getStatusDescription(int status) {
323         try {
324             return STATUS_DESCRIPTIONS[status];
325         } catch (ArrayIndexOutOfBoundsException   e) {
326             return "Unknown status: " + status;
327         }
328     }
329 
330     /**
331      * When you have finished with a Connection you should put it back here.  That will make it available to others.
332      * Unless it's due for expiry, in which case it will... expire
333      */
334     protected void putConnection(ProxyConnectionIF proxyConnection) {
335       
336         if (admin != null) {
337             long now = System.currentTimeMillis();
338             long start = proxyConnection.getTimeLastStartActive();
339             if (now - start < 0) {
340                 log.warn("Future start time detected. #" + proxyConnection.getId() + " start = " + new Date  (start)
341                         + " (" + (now - start) + " milliseconds)");
342             } else if (now - start > 1000000) {
343                 log.warn("Suspiciously long active time. #" + proxyConnection.getId() + " start = " + new Date  (start));
344             }
345             admin.connectionReturned(now - start);
346         }
347 
348         // It's possible that this connection is due for expiry
349         if (proxyConnection.isMarkedForExpiry()) {
350             if (proxyConnection.setStatus(ProxyConnectionIF.STATUS_ACTIVE, ProxyConnectionIF.STATUS_NULL)) {
351                 expireProxyConnection(proxyConnection, proxyConnection.getReasonForMark(), REQUEST_EXPIRY);
352             }
353         } else {
354 
355             // Optionally, test it to see if it is ok
356             if (getDefinition().isTestAfterUse()) {
357                 // It will get removed by this call if it is no good
358                 testConnection(proxyConnection);
359             }
360 
361             // Let's make it available for someone else
362             if (!proxyConnection.setStatus(ProxyConnectionIF.STATUS_ACTIVE, ProxyConnectionIF.STATUS_AVAILABLE)) {
363                 if (proxyConnection.getStatus() == ProxyConnectionIF.STATUS_AVAILABLE) {
364                     // This is *probably* because the connection has been closed twice.
365                     // Although we can't tell for sure. We'll have to refactor this to use
366                     // throw away wrappers to avoid this problem.
367                     log.warn("Unable to close connection " + proxyConnection.getId()
368                             + " - I suspect that it has been closed already. Closing it more"
369                             + " than once is unwise and should be avoided.");
370                 } else {
371                     log.warn("Unable to set status of connection " + proxyConnection.getId()
372                             + " from " + getStatusDescription(ProxyConnectionIF.STATUS_ACTIVE)
373                             + " to " + getStatusDescription(ProxyConnectionIF.STATUS_AVAILABLE)
374                             + " because it's state was " + getStatusDescription(proxyConnection.getStatus()));
375                 }
376             }
377         }
378 
379         if (log.isDebugEnabled() && getDefinition().isVerbose()) {
380             log.debug(displayStatistics() + " - Connection #" + proxyConnection.getId() + " returned (now "
381                     + getStatusDescription(proxyConnection.getStatus()) + ")");
382         }
383 
384     }
385 
386     /** This means that there's something wrong the connection and it's probably best if no one uses it again. */
387     protected void throwConnection(ProxyConnectionIF proxyConnection, String   reason) {
388         expireConnectionAsSoonAsPossible(proxyConnection, reason, true);
389     }
390 
391     /** Get a ProxyConnection by index */
392     private ProxyConnectionIF getProxyConnection(int i) {
393         return (ProxyConnectionIF) proxyConnections.get(i);
394     }
395 
396     /**
397      * Return an array of all the connections
398      * @return array of connections
399      */
400     protected ProxyConnectionIF[] getProxyConnections() {
401         return (ProxyConnectionIF[]) proxyConnections.toArray(new ProxyConnectionIF[proxyConnections.size()]);
402     }
403 
404     /**
405      * Remove a ProxyConnection by calling its {@link ConnectionListenerIF#onDeath onDeath} event,
406      * closing it (for real) and then removing it from the list.
407      * @param proxyConnection the connection to remove
408      * @param reason for log audit
409      * @param forceExpiry true means close now, whether it is active or not; false means if it is active then
410      * merely mark it for expiry so that it is removed as soon as it finished being active
411      * @param triggerSweep if true then this removal will trigger a prototype sweep
412      */
413     protected void removeProxyConnection(ProxyConnectionIF proxyConnection, String   reason, boolean forceExpiry, boolean triggerSweep) {
414         // Just check that it is null
415         if (forceExpiry || proxyConnection.isNull()) {
416 
417             proxyConnection.setStatus(ProxyConnectionIF.STATUS_NULL);
418 
419             /* Run some code everytime we destroy a connection */
420 
421             try {
422                 onDeath(proxyConnection.getConnection());
423             } catch (SQLException   e) {
424                 log.error("Problem during onDeath (ignored)", e);
425             }
426 
427             // The reallyClose() method also decrements the connectionCount.
428             try {
429                 proxyConnection.reallyClose();
430             } catch (SQLException   e) {
431                 log.error(e);
432             }
433 
434             try {
435                 // If we're shutting down then getting a write lock will cause a deadlock
436                 if (isConnectionPoolUp()) {
437                     acquireConnectionStatusWriteLock();
438                 }
439                 proxyConnections.remove(proxyConnection);
440             } finally {
441                 if (isConnectionPoolUp()) {
442                     releaseConnectionStatusWriteLock();
443                 }
444             }
445 
446             if (log.isDebugEnabled()) {
447                 log.debug(displayStatistics() + " - #" + FormatHelper.formatMediumNumber(proxyConnection.getId())
448                         + " removed because " + reason + ".");
449             }
450 
451             if (triggerSweep) {
452                 PrototyperController.triggerSweep(getDefinition().getAlias());
453             }
454 
455         } else {
456             log.error(displayStatistics() + " - #" + FormatHelper.formatMediumNumber(proxyConnection.getId())
457                     + " was not removed because isNull() was false.");
458         }
459     }
460 
461     protected void expireProxyConnection(ProxyConnectionIF proxyConnection, String   reason, boolean forceExpiry) {
462         removeProxyConnection(proxyConnection, reason, forceExpiry, true);
463     }
464 
465     /**
466      * Call this to shutdown gracefully.
467      *  @param delay how long to wait for connections to become free before forcing them to close anyway
468      */
469     protected void shutdown(int delay, String   finalizerName) throws Throwable   {
470 
471         final String   alias = getDefinition().getAlias();
472         try {
473             /* This will stop us giving out any more connections and may
474             cause some of the threads to die. */
475 
476             acquirePrimaryWriteLock();
477 
478             if (connectionPoolUp) {
479 
480                 connectionPoolUp = false;
481                 long startFinalize = System.currentTimeMillis();
482                 shutdownThread = Thread.currentThread();
483 
484                 if (delay > 0) {
485                     log.info("Shutting down '" + alias + "' pool started at "
486                             + dateStarted + " - waiting for " + delay
487                             + " milliseconds for everything to stop.  [ "
488                             + finalizerName + "]");
489                 } else {
490                     log.info("Shutting down '" + alias + "' pool immediately [" + finalizerName + "]");
491                 }
492 
493                 /* Interrupt the threads (in case they're sleeping) */
494 
495                 boolean connectionClosedManually = false;
496                 try {
497 
498                     try {
499                         HouseKeeperController.cancel(alias);
500                     } catch (ProxoolException e) {
501                         log.error("Shutdown couldn't cancel house keeper", e);
502                     }
503 
504                     // Cancel the admin thread (for statistics)
505                     if (admin != null) {
506                         admin.cancelAll();
507                     }
508 
509                     /* Patience, patience. */
510 
511                     if (connectionCountByState[ProxyConnectionIF.STATUS_ACTIVE] != 0) {
512                         long endWait = startFinalize + delay;
513                         LOG.info("Waiting until " + new Date  (endWait) + " for all connections to become inactive (active count is "
514                                 + connectionCountByState[ProxyConnectionIF.STATUS_ACTIVE] + ").");
515                         while (true) {
516                             long timeout = endWait - System.currentTimeMillis();
517                             if (timeout > 0) {
518                                 synchronized (Thread.currentThread()) {
519                                     try {
520                                         Thread.currentThread().wait(timeout);
521                                     } catch (InterruptedException   e) {
522                                         log.debug("Interrupted whilst sleeping.");
523                                     }
524                                 }
525                             }
526                             int activeCount = connectionCountByState[ProxyConnectionIF.STATUS_ACTIVE];
527                             if (activeCount == 0) {
528                                 break;
529                             }
530                             if (System.currentTimeMillis() < endWait) {
531                                 LOG.info("Still waiting for active count to reach zero (currently " + activeCount + ").");
532                             } else {
533                                 // There are still connections active. Oh well, we're not _that_ patient
534                                 LOG.warn("Shutdown waited for "
535                                         + (System.currentTimeMillis() - startFinalize) + " milliseconds for all "
536                                         + "the connections to become inactive but the active count is still "
537                                         + activeCount + ". Shutting down anyway.");
538                                 break;
539                             }
540                             Thread.sleep(100);
541                         }
542                     }
543 
544                     prototyper.cancel();
545                     
546                     // Silently close all connections
547                     for (int i = proxyConnections.size() - 1; i >= 0; i--) {
548                         long id = getProxyConnection(i).getId();
549                         try {
550                             connectionClosedManually = true;
551                             removeProxyConnection(getProxyConnection(i), "of shutdown", true, false);
552                             if (log.isDebugEnabled()) {
553                                 log.debug("Connection #" + id + " closed");
554                             }
555                         } catch (Throwable   t) {
556                             if (log.isDebugEnabled()) {
557                                 log.debug("Problem closing connection #" + id, t);
558                             }
559 
560                         }
561                     }
562 
563                 } catch (Throwable   t) {
564                     log.error("Unknown problem finalizing pool", t);
565                 } finally {
566 
567                     ConnectionPoolManager.getInstance().removeConnectionPool(alias);
568 
569                     if (log.isDebugEnabled()) {
570                         log.info("'" + alias + "' pool has been closed down by " + finalizerName
571                                 + " in " + (System.currentTimeMillis() - startFinalize) + " milliseconds.");
572                         if (!connectionClosedManually) {
573                             log.debug("No connections required manual removal.");
574                         }
575                     }
576                     super.finalize();
577                 }
578             } else {
579                 if (log.isDebugEnabled()) {
580                     log.debug("Ignoring duplicate attempt to shutdown '" + alias + "' pool by " + finalizerName);
581                 }
582             }
583         } catch (Throwable   t) {
584             log.error(finalizerName + " couldn't shutdown pool", t);
585         } finally {
586             releasePrimaryWriteLock();
587         }
588     }
589 
590     /**
591      * You should {@link #acquireConnectionStatusReadLock acquire}
592      * a read lock if you want this to be accurate (but that might have
593      * an impact on the performance of your pool).
594      * @see ConnectionPoolStatisticsIF#getAvailableConnectionCount
595      */
596     public int getAvailableConnectionCount() {
597         return connectionCountByState[ConnectionInfoIF.STATUS_AVAILABLE];
598     }
599 
600     /**
601      * You should {@link #acquireConnectionStatusReadLock acquire}
602      * a read lock if you want this to be accurate (but that might have
603      * an impact on the performance of your pool).
604      * @see ConnectionPoolStatisticsIF#getActiveConnectionCount
605      */
606     public int getActiveConnectionCount() {
607         return connectionCountByState[ConnectionInfoIF.STATUS_ACTIVE];
608     }
609 
610     /**
611      * You should {@link #acquireConnectionStatusReadLock acquire}
612      * a read lock if you want this to be accurate (but that might have
613      * an impact on the performance of your pool).
614      * @see ConnectionPoolStatisticsIF#getOfflineConnectionCount
615      */
616     public int getOfflineConnectionCount() {
617         return connectionCountByState[ConnectionInfoIF.STATUS_OFFLINE];
618     }
619 
620     protected String   displayStatistics() {
621 
622         if (!loggedLegend) {
623             log.info("Proxool statistics legend: \"s - r  (a/t/o)\" > s=served, r=refused (only shown if non-zero), a=active, t=total, o=offline (being tested)");
624             loggedLegend = true;
625         }
626 
627         StringBuffer   statistics = new StringBuffer  ();
628         statistics.append(FormatHelper.formatBigNumber(getConnectionsServedCount()));
629 
630         if (getConnectionsRefusedCount() > 0) {
631             statistics.append(" -");
632             statistics.append(FormatHelper.formatBigNumber(getConnectionsRefusedCount()));
633         }
634 
635         statistics.append(" (");
636         statistics.append(FormatHelper.formatSmallNumber(getActiveConnectionCount()));
637         statistics.append("/");
638         statistics.append(FormatHelper.formatSmallNumber(getAvailableConnectionCount() + getActiveConnectionCount()));
639         statistics.append("/");
640         statistics.append(FormatHelper.formatSmallNumber(getOfflineConnectionCount()));
641         statistics.append(")");
642 
643         // Don't need this triple check any more.
644         /*
645         if (getDefinition().getDebugLevel() == ConnectionPoolDefinitionIF.DEBUG_LEVEL_LOUD) {
646             statistics.append(", cc=");
647             statistics.append(connectionCount);
648             statistics.append(", ccc=");
649             statistics.append(connectedConnectionCount);
650         }
651         */
652 
653         return statistics.toString();
654     }
655 
656     protected void expireAllConnections(String   reason, boolean merciful) {
657 
658         // Do this in two stages because expiring a connection will trigger
659         // the prototyper to make more. And that might mean we end up
660         // killing a newly made connection;
661         Set   pcs = new HashSet  ();
662         for (int i = proxyConnections.size() - 1; i >= 0; i--) {
663             pcs.add(proxyConnections.get(i));
664         }
665 
666         Iterator   i = pcs.iterator();
667         while (i.hasNext()) {
668             ProxyConnectionIF pc = (ProxyConnectionIF) i.next();
669             expireConnectionAsSoonAsPossible(pc, reason, merciful);
670         }
671     }
672 
673     protected void expireConnectionAsSoonAsPossible(ProxyConnectionIF proxyConnection, String   reason, boolean merciful) {
674         if (proxyConnection.setStatus(ProxyConnectionIF.STATUS_AVAILABLE, ProxyConnectionIF.STATUS_OFFLINE)) {
675             if (proxyConnection.setStatus(ProxyConnectionIF.STATUS_OFFLINE, ProxyConnectionIF.STATUS_NULL)) {
676                 // It is.  Expire it now .
677                 expireProxyConnection(proxyConnection, reason, REQUEST_EXPIRY);
678             }
679         } else {
680             // Oh no, it's in use.
681 
682             if (merciful) {
683                 //Never mind, we'll mark it for expiry
684                 // next time it is available.  This will happen in the
685                 // putConnection() method.
686                 proxyConnection.markForExpiry(reason);
687                 if (log.isDebugEnabled()) {
688                     log.debug(displayStatistics() + " - #" + FormatHelper.formatMediumNumber(proxyConnection.getId()) + " marked for expiry.");
689                 }
690             } else {
691                 // So? Kill, kill, kill
692 
693                 // We have to make sure it's null first.
694                 expireProxyConnection(proxyConnection, reason, FORCE_EXPIRY);
695             }
696 
697         } // END if (proxyConnection.setOffline())
698     }
699 
700     protected void registerRemovedConnection(int status) {
701         prototyper.connectionRemoved();
702         connectionCountByState[status]--;
703     }
704 
705     /**
706      * You should {@link #acquireConnectionStatusWriteLock acquire} a write lock
707      * before calling this method
708      * @param oldStatus so we know which count to decrement
709      * @param newStatus so we know which count to increment
710      */
711     protected void changeStatus(int oldStatus, int newStatus) {
712         // LOG.debug("About to change status");
713         connectionCountByState[oldStatus]--;
714         connectionCountByState[newStatus]++;
715         // LOG.debug("Changing status from " + oldStatus + " to " + newStatus);
716         // Check to see if shutdown is waiting for all connections to become
717         // non-active
718         if (shutdownThread != null && connectionCountByState[ProxyConnectionIF.STATUS_ACTIVE] == 0) {
719             synchronized (shutdownThread) {
720                 shutdownThread.notify();
721             }
722         }
723 
724     }
725 
726     public long getConnectionsServedCount() {
727         return connectionsServedCount;
728     }
729 
730     public long getConnectionsRefusedCount() {
731         return connectionsRefusedCount;
732     }
733 
734     protected ConnectionPoolDefinition getDefinition() {
735         return definition;
736     }
737 
738     /**
739      * Changes both the way that any new connections will be made, and the behaviour of the pool. Consider
740      * calling expireAllConnections() if you're in a hurry.
741      */
742     protected synchronized void setDefinition(ConnectionPoolDefinition definition) throws ProxoolException {
743         this.definition = definition;
744 
745         try {
746             Class.forName(definition.getDriver());
747         } catch (ClassNotFoundException   e) {
748             log.error("Couldn't load class " + definition.getDriver(), e);
749             throw new ProxoolException("Couldn't load class " + definition.getDriver());
750         } catch (NullPointerException   e) {
751    &nbs