BasicResourcePool


1   /*
2    * Distributed as part of c3p0 v.0.9.1
3    *
4    * Copyright (C) 2005 Machinery For Change, Inc.
5    *
6    * Author: Steve Waldman <swaldman@mchange.com>
7    *
8    * This library is free software; you can redistribute it and/or modify
9    * it under the terms of the GNU Lesser General Public License version 2.1, as 
10   * published by the Free Software Foundation.
11   *
12   * This software is distributed in the hope that it will be useful,
13   * but WITHOUT ANY WARRANTY; without even the implied warranty of
14   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15   * GNU Lesser General Public License for more details.
16   *
17   * You should have received a copy of the GNU Lesser General Public License
18   * along with this software; see the file LICENSE.  If not, write to the
19   * Free Software Foundation, Inc., 59 Temple Place, Suite 330,
20   * Boston, MA 02111-1307, USA.
21   */
22  
23  
24  package com.mchange.v2.resourcepool;
25  
26  import java.util.*;
27  import com.mchange.v2.async.*;
28  import com.mchange.v2.log.*;
29  import com.mchange.v2.lang.ThreadUtils;
30  import com.mchange.v2.util.ResourceClosedException;
31  
32  class BasicResourcePool implements ResourcePool
33  {
34      private final static MLogger logger = MLog.getLogger( BasicResourcePool.class );
35  
36      final static int AUTO_CULL_FREQUENCY_DIVISOR = 4;
37      final static int AUTO_MAX_CULL_FREQUENCY = (15 * 60 * 1000); //15 mins
38      final static int AUTO_MIN_CULL_FREQUENCY = (1 * 1000); //15 mins
39  
40  
41      //XXX: temporary -- for selecting between AcquireTask types
42      //     remove soon, and use only ScatteredAcquireTask,
43      //     presuming no problems appear
44      final static String   USE_SCATTERED_ACQUIRE_TASK_KEY = "com.mchange.v2.resourcepool.experimental.useScatteredAcquireTask";
45      final static boolean USE_SCATTERED_ACQUIRE_TASK;
46      static
47      {
48          String   checkScattered = com.mchange.v2.cfg.MultiPropertiesConfig.readVmConfig().getProperty(USE_SCATTERED_ACQUIRE_TASK_KEY);
49          if (checkScattered != null && checkScattered.trim().toLowerCase().equals("true"))
50          {
51              USE_SCATTERED_ACQUIRE_TASK = true;
52              if ( logger.isLoggable( MLevel.INFO ) )
53                  logger.info(BasicResourcePool.class.getName() + " using experimental ScatteredAcquireTask.");
54          }
55          else
56              USE_SCATTERED_ACQUIRE_TASK = false;
57      }
58      // end temporary switch between acquire task types
59  
60      //MT: unchanged post c'tor
61      final Manager mgr;
62  
63      final int start;
64      final int min;
65      final int max;
66      final int inc;
67  
68      final int num_acq_attempts;
69      final int acq_attempt_delay;
70  
71      final long check_idle_resources_delay;       //milliseconds
72      final long max_resource_age;                 //milliseconds
73      final long max_idle_time;                    //milliseconds
74      final long excess_max_idle_time;             //milliseconds
75      final long destroy_unreturned_resc_time;     //milliseconds
76      final long expiration_enforcement_delay;     //milliseconds
77  
78      final boolean break_on_acquisition_failure;
79      final boolean debug_store_checkout_exceptions;
80  
81      final long pool_start_time = System.currentTimeMillis();
82  
83      //MT: not-reassigned, thread-safe, and independent
84      final BasicResourcePoolFactory factory;
85      final AsynchronousRunner       taskRunner;
86      final RunnableQueue            asyncEventQueue;
87      final ResourcePoolEventSupport rpes;
88  
89      //MT: protected by this' lock
90      Timer                    cullAndIdleRefurbishTimer;
91      TimerTask                cullTask;
92      TimerTask                idleRefurbishTask;
93      HashSet                  acquireWaiters = new HashSet();
94      HashSet                  otherWaiters = new HashSet();
95  
96      int pending_acquires;
97      int pending_removes;
98  
99      int target_pool_size;
100 
101     /*  keys are all valid, managed resources, value is a PunchCard */ 
102     HashMap  managed = new HashMap();
103 
104     /* all valid, managed resources currently available for checkout */
105     LinkedList unused = new LinkedList();
106 
107     /* resources which have been invalidated somehow, but which are */
108     /* still checked out and in use.                                */
109     HashSet  excluded = new HashSet();
110 
111     Map formerResources = new WeakHashMap();
112 
113     Set idleCheckResources = new HashSet();
114 
115     boolean force_kill_acquires = false;
116 
117     boolean broken = false;
118 
119 //  long total_acquired = 0;
120 
121     long failed_checkins   = 0;
122     long failed_checkouts  = 0;
123     long failed_idle_tests = 0;
124 
125     Throwable   lastCheckinFailure      = null;
126     Throwable   lastCheckoutFailure     = null;
127     Throwable   lastIdleTestFailure     = null;
128     Throwable   lastResourceTestFailure = null;
129 
130     Throwable   lastAcquisitionFailiure = null;
131 
132     //DEBUG only!
133     Object   exampleResource;
134 
135     public long getStartTime()
136     { return pool_start_time; }
137 
138     public long getUpTime()
139     { return System.currentTimeMillis() - pool_start_time; }
140 
141     public synchronized long getNumFailedCheckins()
142     { return failed_checkins; }
143 
144     public synchronized long getNumFailedCheckouts()
145     { return failed_checkouts; }
146 
147     public synchronized long getNumFailedIdleTests()
148     { return failed_idle_tests; }
149 
150     public synchronized Throwable   getLastCheckinFailure()
151     { return lastCheckinFailure; }
152 
153     //must be called from a pre-existing sync'ed block
154     private void setLastCheckinFailure(Throwable   t)
155     {
156         assert ( Thread.holdsLock(this));
157 
158         this.lastCheckinFailure = t;
159         this.lastResourceTestFailure = t;
160     }
161 
162     public synchronized Throwable   getLastCheckoutFailure()
163     { return lastCheckoutFailure; }
164 
165     //must be called from a pre-existing sync'ed block
166     private void setLastCheckoutFailure(Throwable   t)
167     {
168         assert ( Thread.holdsLock(this));
169 
170         this.lastCheckoutFailure = t;
171         this.lastResourceTestFailure = t;
172     }
173 
174     public synchronized Throwable   getLastIdleCheckFailure()
175     { return lastIdleTestFailure; }
176 
177     //must be called from a pre-existing sync'ed block
178     private void setLastIdleCheckFailure(Throwable   t)
179     {
180         assert ( Thread.holdsLock(this));
181 
182         this.lastIdleTestFailure = t;
183         this.lastResourceTestFailure = t;
184     }
185 
186     public synchronized Throwable   getLastResourceTestFailure()
187     { return lastResourceTestFailure; }
188 
189     public synchronized Throwable   getLastAcquisitionFailure()
190     { return lastAcquisitionFailiure; }
191 
192     // ought not be called while holding this' lock
193     private synchronized void setLastAcquisitionFailure( Throwable   t )
194     { this.lastAcquisitionFailiure = t; }
195 
196     public synchronized int getNumCheckoutWaiters()
197     { return acquireWaiters.size(); }
198 
199     private void addToFormerResources( Object   resc )
200     { formerResources.put( resc, null ); }
201 
202     private boolean isFormerResource( Object   resc )
203     { return formerResources.keySet().contains( resc ); }
204 
205     /**
206      * @param factory may be null
207      */
208     public BasicResourcePool(Manager                  mgr, 
209                     int                      start,
210                     int                      min, 
211                     int                      max, 
212                     int                      inc,
213                     int                      num_acq_attempts,
214                     int                      acq_attempt_delay,
215                     long                     check_idle_resources_delay,
216                     long                     max_resource_age,
217                     long                     max_idle_time,
218                     long                     excess_max_idle_time,
219                     long                     destroy_unreturned_resc_time,
220                     long                     expiration_enforcement_delay,
221                     boolean                  break_on_acquisition_failure,
222                     boolean                  debug_store_checkout_exceptions,
223                     AsynchronousRunner       taskRunner,
224                     RunnableQueue            asyncEventQueue,
225                     Timer                    cullAndIdleRefurbishTimer,
226                     BasicResourcePoolFactory factory)
227     throws ResourcePoolException
228     {
229         try
230         {
231             this.mgr                              = mgr;
232             this.start                            = start;
233             this.min                              = min;
234             this.max                              = max;
235             this.inc                              = inc;
236             this.num_acq_attempts                 = num_acq_attempts;
237             this.acq_attempt_delay                = acq_attempt_delay;
238             this.check_idle_resources_delay       = check_idle_resources_delay;
239             this.max_resource_age                 = max_resource_age;
240             this.max_idle_time                    = max_idle_time;
241             this.excess_max_idle_time             = excess_max_idle_time;
242             this.destroy_unreturned_resc_time     = destroy_unreturned_resc_time;
243             //this.expiration_enforcement_delay     = expiration_enforcement_delay; -- set up below
244             this.break_on_acquisition_failure     = break_on_acquisition_failure;
245             this.debug_store_checkout_exceptions  = (debug_store_checkout_exceptions && destroy_unreturned_resc_time > 0);
246             this.taskRunner                       = taskRunner;
247             this.asyncEventQueue                  = asyncEventQueue;
248             this.cullAndIdleRefurbishTimer        = cullAndIdleRefurbishTimer;
249             this.factory                          = factory;
250 
251             this.pending_acquires = 0;
252             this.pending_removes  = 0;
253 
254             this.target_pool_size = Math.max(start, min);
255 
256             if (asyncEventQueue != null)
257                 this.rpes = new ResourcePoolEventSupport(this);
258             else
259                 this.rpes = null;
260 
261             //start acquiring our initial resources
262             ensureStartResources();
263 
264             if (mustEnforceExpiration())
265             {
266                 if (expiration_enforcement_delay <= 0)
267                     this.expiration_enforcement_delay = automaticExpirationEnforcementDelay();
268                 else
269                     this.expiration_enforcement_delay = expiration_enforcement_delay;
270 
271                 this.cullTask = new CullTask();
272                 //System.err.println("minExpirationTime(): " + minExpirationTime());
273                 //System.err.println("this.expiration_enforcement_delay: " + this.expiration_enforcement_delay);
274                 cullAndIdleRefurbishTimer.schedule( cullTask, minExpirationTime(), this.expiration_enforcement_delay );
275             }
276             else
277                 this.expiration_enforcement_delay = expiration_enforcement_delay;
278 
279             //System.err.println("this.check_idle_resources_delay: " + this.check_idle_resources_delay);
280             if (check_idle_resources_delay > 0)
281             {
282                 this.idleRefurbishTask = new CheckIdleResourcesTask();
283                 cullAndIdleRefurbishTimer.schedule( idleRefurbishTask, 
284                                 check_idle_resources_delay, 
285                                 check_idle_resources_delay );
286             }
287 
288             if ( logger.isLoggable( MLevel.FINER ) )
289                 logger.finer( this + " config: [start -> " + this.start + "; min -> " + this.min + "; max -> " + this.max + "; inc -> " + this.inc +
290                                 "; num_acq_attempts -> " + this.num_acq_attempts + "; acq_attempt_delay -> " + this.acq_attempt_delay +
291                                 "; check_idle_resources_delay -> " + this.check_idle_resources_delay + "; mox_resource_age -> " + this.max_resource_age +
292                                 "; max_idle_time -> " + this.max_idle_time + "; excess_max_idle_time -> " + this.excess_max_idle_time +
293                                 "; destroy_unreturned_resc_time -> " + this.destroy_unreturned_resc_time +
294                                 "; expiration_enforcement_delay -> " + this.expiration_enforcement_delay + 
295                                 "; break_on_acquisition_failure -> " + this.break_on_acquisition_failure + 
296                                 "; debug_store_checkout_exceptions -> " + this.debug_store_checkout_exceptions + 
297                 "]");
298 
299         }
300         catch (Exception   e)
301         {
302 //          if ( logger.isLoggable( MLevel.WARNING) )
303 //          logger.log( MLevel.WARNING, "Could not create resource pool due to Exception!", e );
304 
305             throw ResourcePoolUtils.convertThrowable( e ); 
306         }
307     }
308 
309 //  private boolean timerRequired()
310 //  { return mustEnforceExpiration() || mustTestIdleResources(); }
311 
312     // no need to sync
313     private boolean mustTestIdleResources()
314     { return check_idle_resources_delay > 0; }
315 
316     // no need to sync
317     private boolean mustEnforceExpiration()
318     {
319         return 
320         max_resource_age > 0 ||
321         max_idle_time > 0 ||
322         excess_max_idle_time > 0 ||
323         destroy_unreturned_resc_time > 0;
324     }
325 
326     // no need to sync
327     private long minExpirationTime()
328     {
329         long out = Long.MAX_VALUE;
330         if (max_resource_age > 0)
331             out = Math.min( out, max_resource_age );
332         if (max_idle_time > 0)
333             out = Math.min( out, max_idle_time );
334         if (excess_max_idle_time > 0)
335             out = Math.min( out, excess_max_idle_time );
336         if (destroy_unreturned_resc_time > 0)
337             out = Math.min( out, expiration_enforcement_delay );
338         return out;
339     }
340 
341     private long automaticExpirationEnforcementDelay()
342     {
343         long out = minExpirationTime();
344         out /= AUTO_CULL_FREQUENCY_DIVISOR;
345         out = Math.min( out, AUTO_MAX_CULL_FREQUENCY );
346         out = Math.max( out, AUTO_MIN_CULL_FREQUENCY );
347         return out;
348     }
349 
350     public long getEffectiveExpirationEnforcementDelay()
351     { return expiration_enforcement_delay; }
352 
353     private synchronized boolean isBroken()
354     { return broken; }
355 
356     // no need to sync
357     private boolean supportsEvents()
358     { return asyncEventQueue != null; }
359 
360     public Object   checkoutResource() 
361     throws ResourcePoolException, InterruptedException  
362     {
363         try { return checkoutResource( 0 ); }
364         catch (TimeoutException e)
365         {
366             //this should never happen
367             //e.printStackTrace();
368             if ( logger.isLoggable( MLevel.WARNING ) )
369                 logger.log( MLevel.WARNING, "Huh??? TimeoutException with no timeout set!!!", e);
370 
371             throw new ResourcePoolException("Huh??? TimeoutException with no timeout set!!!", e);
372         }
373     }
374 
375     // must be called from synchronized method, idempotent
376     private void _recheckResizePool()
377     {
378         assert Thread.holdsLock(this);
379 
380         if (! broken)
381         {
382             int msz = managed.size();
383             //int expected_size = msz + pending_acquires - pending_removes;
384 
385 //          System.err.print("target: " + target_pool_size);
386 //          System.err.println(" (msz: " + msz + "; pending_acquires: " + pending_acquires + "; pending_removes: " + pending_removes + ')');
387             //new Exception( "_recheckResizePool() STACK TRACE" ).printStackTrace();
388 
389             int shrink_count;
390             int expand_count;
391 
392             if ((shrink_count = msz - pending_removes - target_pool_size) > 0)
393                 shrinkPool( shrink_count );
394             else if ((expand_count = target_pool_size - (msz + pending_acquires)) > 0)
395                 expandPool( expand_count );
396         }
397     }
398 
399     private synchronized void incrementPendingAcquires()
400     { 
401         ++pending_acquires; 
402 
403         if (logger.isLoggable(MLevel.FINEST))
404             logger.finest("incremented pending_acquires: " + pending_acquires);
405         //new Exception("ACQUIRE SOURCE STACK TRACE").printStackTrace();
406     }
407 
408     private synchronized void incrementPendingRemoves()
409     { 
410         ++pending_removes; 
411 
412         if (logger.isLoggable(MLevel.FINEST))
413             logger.finest("incremented pending_removes: " + pending_removes);
414         //new Exception("REMOVE SOURCE STACK TRACE").printStackTrace();
415     }
416 
417     private synchronized void decrementPendingAcquires()
418     { 
419         --pending_acquires; 
420 
421         if (logger.isLoggable(MLevel.FINEST))
422             logger.finest("decremented pending_acquires: " + pending_acquires);
423         //new Exception("ACQUIRE SOURCE STACK TRACE").printStackTrace();
424     }
425 
426     private synchronized void decrementPendingRemoves()
427     { 
428         --pending_removes; 
429 
430         if (logger.isLoggable(MLevel.FINEST))
431             logger.finest("decremented pending_removes: " + pending_removes);
432         //new Exception("ACQUIRE SOURCE STACK TRACE").printStackTrace();
433     }
434 
435     // idempotent
436     private synchronized void recheckResizePool()
437     { _recheckResizePool(); }
438 
439     // must be called from synchronized method
440     private void expandPool(int count)
441     {
442         assert Thread.holdsLock(this);
443 
444         // XXX: temporary switch -- assuming no problems appear, we'll get rid of AcquireTask
445         //      in favor of ScatteredAcquireTask
446         if ( USE_SCATTERED_ACQUIRE_TASK )
447         {
448             for (int i = 0; i < count; ++i)
449                 taskRunner.postRunnable( new ScatteredAcquireTask() );
450         }
451         else
452         {
453             for (int i = 0; i < count; ++i)
454                 taskRunner.postRunnable( new AcquireTask() );
455         }
456     }
457 
458     // must be called from synchronized method
459     private void shrinkPool(int count)
460     {
461         assert Thread.holdsLock(this);
462 
463         for (int i = 0; i < count; ++i)
464             taskRunner.postRunnable( new RemoveTask() ); 
465     }
466 
467     /*
468      * This function recursively calls itself... under nonpathological
469      * situations, it shouldn't be a problem, but if resources can never
470      * successfully check out for some reason, we might blow the stack...
471      *
472      * by the semantics of wait(), a timeout of zero means forever.
473      */
474     public Object   checkoutResource( long timeout )
475     throws TimeoutException, ResourcePoolException, InterruptedException  
476     {
477         Object   resc = prelimCheckoutResource( timeout );
478 
479         boolean refurb = attemptRefurbishResourceOnCheckout( resc );
480 
481         synchronized( this )
482         {
483             if (!refurb)
484             {
485                 removeResource( resc );
486                 ensureMinResources();
487                 resc = null;
488             }
489             else
490             {
491                 asyncFireResourceCheckedOut( resc, managed.size(), unused.size(), excluded.size() );
492                 if (Debug.DEBUG && Debug.TRACE == Debug.TRACE_MAX) trace();
493 
494                 PunchCard card = (PunchCard) managed.get( resc );
495                 if (card == null) //the resource has been removed!
496                 {
497                     if (logger.isLoggable( MLevel.FINE ))
498                         logger.fine("Resource " + resc + " was removed from the pool while it was being checked out " +
499                         " or refurbished for checkout.");
500                     resc = null;
501                 }
502                 else
503                 {
504                     card.checkout_time = System.currentTimeMillis();
505                     if (debug_store_checkout_exceptions)
506                         card.checkoutStackTraceException = new Exception  ("DEBUG ONLY: Overdue resource check-out stack trace.");
507                 }
508             }
509         }
510 
511         // best to do the recheckout while we don't hold this'
512         // lock, so we don't refurbish-on-checkout while holding.
513         if (resc == null)
514             return checkoutResource( timeout );
515         else
516             return resc;
517     }
518 
519     private synchronized Object   prelimCheckoutResource( long timeout )
520     throws TimeoutException, ResourcePoolException, InterruptedException  
521     {
522         try
523         {
524             ensureNotBroken();
525 
526             int available = unused.size();
527             if (available == 0)
528             {
529                 int msz = managed.size();
530 
531                 if (msz < max)
532                 {
533                     // to cover all the load, we need the current size, plus those waiting already for acquisition, 
534                     // plus the current client 
535                     int desired_target = msz + acquireWaiters.size() + 1;
536 
537                     if (logger.isLoggable(MLevel.FINER))
538                         logger.log(MLevel.FINER, "acquire test -- pool size: " + msz + "; target_pool_size: " + target_pool_size + "; desired target? " + desired_target);
539 
540                     if (desired_target >= target_pool_size)
541                     {
542                         //make sure we don't grab less than inc Connections at a time, if we can help it.
543                         desired_target = Math.max(desired_target, target_pool_size + inc);
544 
545                         //make sure our target is within its bounds
546                         target_pool_size = Math.max( Math.min( max, desired_target ), min );
547 
548                         _recheckResizePool();
549                     }
550                 }
551                 else
552                 {
553                     if (logger.isLoggable(MLevel.FINER))
554                         logger.log(MLevel.FINER, "acquire test -- pool is already maxed out. [managed: " + msz + "; max: " + max + "]");
555                 }
556 
557                 awaitAvailable(timeout); //throws timeout exception
558             }
559 
560             Object    resc = unused.get(0);
561 
562             // this is a hack -- but "doing it right" adds a lot of complexity, and collisions between
563             // an idle check and a checkout should be relatively rare. anyway, it should work just fine.
564             if ( idleCheckResources.contains( resc ) )
565             {
566                 if (Debug.DEBUG && logger.isLoggable( MLevel.FINER))
567                     logger.log( MLevel.FINER, 
568                                     "Resource we want to check out is in idleCheck! (waiting until idle-check completes.) [" + this + "]");
569 
570                 // we'll move remove() to after the if, so we don't have to add back
571                 // unused.add(0, resc );
572 
573                 // we'll wait for "something to happen" -- probably an idle check to
574                 // complete -- then we'll try again and hope for the best.
575                 Thread   t = Thread.currentThread();
576                 try
577                 {
578                     otherWaiters.add ( t );
579                     this.wait( timeout );
580                     ensureNotBroken();
581                 }
582                 finally
583                 { otherWaiters.remove( t ); }
584                 return prelimCheckoutResource( timeout );
585             }
586             else if ( shouldExpire( resc ) )
587             {
588                 removeResource( resc );
589                 ensureMinResources();
590                 return prelimCheckoutResource( timeout );
591             }
592             else
593             {
594                 unused.remove(0);
595                 return resc;
596             }
597         }
598         catch ( ResourceClosedException e ) // one of our async threads died
599         {
600             //System.err.println(this + " -- the pool was found to be closed or broken during an attempt to check out a resource.");
601             //e.printStackTrace();
602             if (logger.isLoggable( MLevel.SEVERE ))
603                 logger.log( MLevel.SEVERE, this + " -- the pool was found to be closed or broken during an attempt to check out a resource.", e );
604 
605             this.unexpectedBreak();
606             throw e;
607         }
608         catch ( InterruptedException   e )
609         {
610             //      System.err.println(this + " -- an attempt to checkout a resource was interrupted: some other thread " +
611             //                 "must have either interrupted the Thread attempting checkout, or close() was called on the pool.");
612             //      e.printStackTrace();
613             if (broken)
614             {
615                 if (logger.isLoggable( MLevel.FINER ))
616                     logger.log(MLevel.FINER, 
617                                     this + " -- an attempt to checkout a resource was interrupted, because the pool is now closed. " +
618                                     "[Thread: " + Thread.currentThread().getName() + ']',
619                                     e );
620                 else if (logger.isLoggable( MLevel.INFO ))
621                     logger.log(MLevel.INFO, 
622                                     this + " -- an attempt to checkout a resource was interrupted, because the pool is now closed. " +
623                                     "[Thread: " + Thread.currentThread().getName() + ']');
624             }
625             else
626             {
627                 if (logger.isLoggable( MLevel.WARNING ))
628                 {
629                     logger.log(MLevel.WARNING, 
630                                     this + " -- an attempt to checkout a resource was interrupted, and the pool is still live: some other thread " +
631                                     "must have either interrupted the Thread attempting checkout!",
632                                     e );
633                 }
634             }
635             throw e;
636         }
637     }
638 
639     public synchronized void checkinResource( Object   resc ) 
640     throws ResourcePoolException
641     {
642         try
643         {
644             //we permit straggling resources to be checked in 
645             //without exception even if we are broken
646             if (managed.keySet().contains(resc))
647                 doCheckinManaged( resc );
648             else if (excluded.contains(resc))
649                 doCheckinExcluded( resc );
650             else if ( isFormerResource(resc) )
651             {
652                 if ( logger.isLoggable( MLevel.FINER ) )
653                     logger.finer("Resource " + resc + " checked-in after having been checked-in already, or checked-in after " +
654                     " having being destroyed for being checked-out too long.");
655             }
656             else
657                 throw new ResourcePoolException("ResourcePool" + (broken ? " [BROKEN!]" : "") + ": Tried to check-in a foreign resource!");
658             if (Debug.DEBUG && Debug.TRACE == Debug.TRACE_MAX) trace();
659         }
660         catch ( ResourceClosedException e ) // one of our async threads died
661         {
662 //          System.err.println(this + 
663 //          " - checkinResource( ... ) -- even broken pools should allow checkins without exception. probable resource pool bug.");
664 //          e.printStackTrace();
665 
666             if ( logger.isLoggable( MLevel.SEVERE ) )
667                 logger.log( MLevel.SEVERE, 
668                                 this + " - checkinResource( ... ) -- even broken pools should allow checkins without exception. probable resource pool bug.", 
669                                 e);
670 
671             this.unexpectedBreak();
672             throw e;
673         }
674     }
675 
676     public synchronized void checkinAll()
677     throws ResourcePoolException
678     {
679         try
680         {
681             Set checkedOutNotExcluded = new HashSet( managed.keySet() );
682             checkedOutNotExcluded.removeAll( unused );
683             for (Iterator ii = checkedOutNotExcluded.iterator(); ii.hasNext(); )
684                 doCheckinManaged( ii.next() );
685             for (Iterator ii = excluded.iterator(); ii.hasNext(); )
686                 doCheckinExcluded( ii.next() );
687         }
688         catch ( ResourceClosedException e ) // one of our async threads died
689         {
690 //          System.err.println(this + 
691 //          " - checkinAll() -- even broken pools should allow checkins without exception. probable resource pool bug.");
692 //          e.printStackTrace();
693 
694             if ( logger.isLoggable( MLevel.SEVERE ) )
695                 logger.log( MLevel.SEVERE,
696                                 this + " - checkinAll() -- even broken pools should allow checkins without exception. probable resource pool bug.",
697                                 e );
698 
699             this.unexpectedBreak();
700             throw e;
701         }
702     }
703 
704     public synchronized int statusInPool( Object   resc )
705     throws ResourcePoolException
706     {
707         try
708         {
709             if ( unused.contains( resc ) )
710                 return KNOWN_AND_AVAILABLE;
711             else if ( managed.keySet().contains( resc ) || excluded.contains( resc ) )
712                 return KNOWN_AND_CHECKED_OUT;
713             else
714                 return UNKNOWN_OR_PURGED;
715         }
716         catch ( ResourceClosedException e ) // one of our async threads died
717         {
718 //          e.printStackTrace();
719             if ( logger.isLoggable( MLevel.SEVERE ) )
720                 logger.log( MLevel.SEVERE, "Apparent pool break.", e );
721             this.unexpectedBreak();
722             throw e;
723         }
724     }
725 
726     public synchronized void markBroken(Object   resc) 
727     {
728         try
729         { 
730             if (Debug.DEBUG && Debug.TRACE == Debug.TRACE_MAX && logger.isLoggable( MLevel.FINER ))
731                 logger.log( MLevel.FINER, "Resource " + resc + " marked broken by pool (" + this + ").");
732 
733             _markBroken( resc ); 
734             ensureMinResources();
735         }
736         catch ( ResourceClosedException e ) // one of our async threads died
737         {
738             //e.printStackTrace();
739             if ( logger.isLoggable( MLevel.SEVERE ) )
740                 logger.log( MLevel.SEVERE, "Apparent pool break.", e );
741             this.unexpectedBreak();
742         }
743     }
744 
745     //min is immutable, no need to synchronize
746     public int getMinPoolSize()
747     { return min; }
748 
749     //max is immutable, no need to synchronize
750     public int getMaxPoolSize()
751     { return max; }
752 
753     public synchronized int getPoolSize()
754     throws ResourcePoolException
755     { return managed.size(); }
756 
757 //  //i don't think i like the async, no-guarantees approach
758 //  public synchronized void requestResize( int req_sz )
759 //  {
760 //  if (req_sz > max)
761 //  req_sz = max;
762 //  else if (req_sz < min)
763 //  req_sz = min;
764 //  int sz = managed.size();
765 //  if (req_sz > sz)
766 //  postAcquireUntil( req_sz );
767 //  else if (req_sz < sz)
768 //  postRemoveTowards( req_sz );
769 //  }
770 
771     public synchronized int getAvailableCount()
772     { return unused.size(); }
773 
774     public synchronized int getExcludedCount()
775     { return excluded.size(); }
776 
777     public synchronized int getAwaitingCheckinCount()
778     { return managed.size() - unused.size() + excluded.size(); }
779 
780     public synchronized void resetPool()
781     {
782         try
783         {
784             for (Iterator ii = cloneOfManaged().keySet().iterator(); ii.hasNext();)
785                 markBrokenNoEnsureMinResources(ii.next());
786             ensureMinResources();
787         }
788         catch ( ResourceClosedException e ) // one of our async threads died
789         {
790             //e.printStackTrace();
791             if ( logger.isLoggable( MLevel.SEVERE ) )
792                 logger.log( MLevel.SEVERE, "Apparent pool break.", e );
793             this.unexpectedBreak();
794         }
795     }
796 
797     public synchronized void close() 
798     throws ResourcePoolException
799     {
800         //we permit closes when we are already broken, so
801         //that resources that were checked out when the break
802         //occured can still be cleaned up
803         close( true );
804     }
805 
806     public void finalize() throws Throwable  
807     {
808         //obviously, clients mustn't rely on finalize,
809         //but must close pools ASAP after use.
810         //System.err.println("finalizing..." + this);
811 
812         if (! broken )
813             this.close();
814     }
815 
816     //no need to sync
817     public void addResourcePoolListener(ResourcePoolListener rpl)
818     { 
819         if ( ! supportsEvents() )
820             throw new RuntimeException  (this + " does not support ResourcePoolEvents. " +
821             "Probably it was constructed by a BasicResourceFactory configured not to support such events.");
822         else
823             rpes.addResourcePoolListener(rpl); 
824     }
825 
826     //no need to sync
827     public void removeResourcePoolListener(ResourcePoolListener rpl)
828     { 
829         if ( ! supportsEvents() )
830             throw new RuntimeException  (this + " does not support ResourcePoolEvents. " +
831             "Probably it was constructed by a BasicResourceFactory configured not to support such events.");
832         else
833             rpes.removeResourcePoolListener(rpl); 
834     }
835 
836     private synchronized boolean isForceKillAcquiresPending()
837     { return force_kill_acquires; }
838 
839     // this is designed as a response to a determination that our resource source is down.
840     // rather than declaring ourselves broken in this case (as we did previously), we
841     // kill all pending acquisition attempts, but retry on new acqusition requests.
842     private synchronized void forceKillAcquires() throws InterruptedException  
843     {
844         Thread   t = Thread.currentThread();
845 
846         try
847         {
848             force_kill_acquires = true;
849             this.notifyAll(); //wake up any threads waiting on an acquire, and force them all to die.
850             while (acquireWaiters.size() > 0) //we want to let all the waiting acquires die before we unset force_kill_acquires
851             {
852                 otherWaiters.add( t ); 
853                 this.wait();
854             }
855             force_kill_acquires = false;
856         }
857         finally
858         { otherWaiters.remove( t ); }
859     }
860 
861     //same as close(), but we do not destroy checked out
862     //resources
863     private synchronized void unexpectedBreak()
864     {
865         if ( logger.isLoggable( MLevel.SEVERE ) )
866             logger.log( MLevel.SEVERE, this + " -- Unexpectedly broken!!!", new ResourcePoolException("Unexpected Break Stack Trace!") );
867         close( false );
868     }
869 
870     // no need to sync
871     private boolean canFireEvents()
872     { return ( asyncEventQueue != null && !isBroken() ); }
873 
874     // no need to sync
875     private void asyncFireResourceAcquired( final Object         resc,
876                     final int          pool_size,
877                     final int          available_size,
878                     final int          removed_but_unreturned_size )
879     {
880         if ( canFireEvents() )
881         {
882             Runnable   r = new Runnable  ()
883             {
884                 public void run()
885                 {rpes.fireResourceAcquired(resc, pool_size, available_size, removed_but_unreturned_size);}
886             };
887             asyncEventQueue.postRunnable(r);
888         }
889     }
890 
891     // no need to sync
892     private void asyncFireResourceCheckedIn( final Object         resc,
893                     final int          pool_size,
894                     final int          available_size,
895                     final int          removed_but_unreturned_size )
896     {
897         if ( canFireEvents() )
898         {
899             Runnable   r = new Runnable  ()
900             {
901                 public void run()
902                 {rpes.fireResourceCheckedIn(resc, pool_size, available_size, removed_but_unreturned_size);}
903             };
904             asyncEventQueue.postRunnable(r);
905         }
906     }
907 
908     // no need to sync
909     private void asyncFireResourceCheckedOut( final Object         resc,
910                     final int          pool_size,
911                     final int          available_size,
912                     final int          removed_but_unreturned_size )
913     {
914         if ( canFireEvents() )
915         {
916             Runnable   r = new Runnable  ()
917             {
918                 public void run()
919                 {rpes.fireResourceCheckedOut(resc,pool_size,available_size,removed_but_unreturned_size);}
920             };
921             asyncEventQueue.postRunnable(r);
922         }
923     }
924 
925     // no need to sync
926     private void asyncFireResourceRemoved( final Object         resc,
927                     final boolean      checked_out_resource,
928                     final int          pool_size,
929                     final int          available_size,
930                     final int          removed_but_unreturned_size )
931     {
932         if ( canFireEvents() )
933         {
934             //System.err.println("ASYNC RSRC REMOVED");
935             //new Exception().printStackTrace();
936             Runnable   r = new Runnable  ()
937             {
938                 public void run()
939                 {
940                     rpes.fireResourceRemoved(resc, checked_out_resource,
941                                     pool_size,available_size,removed_but_unreturned_size);
942                 }
943             };
944             asyncEventQueue.postRunnable(r);
945         }
946     }
947 
948     // needn't be called from a sync'ed method
949     private void destroyResource(final Object   resc)
950     { destroyResource( resc, false ); }
951 
952     // needn't be called from a sync'ed method
953     private void destroyResource(final Object   resc, boolean synchronous)
954     {
955         class DestroyResourceTask implements Runnable  
956         {
957             public void run()
958             {
959                 try 
960                 { 
961                     if (Debug.DEBUG && Debug.TRACE == Debug.TRACE_MAX && logger.isLoggable( MLevel.FINER ))
962                         logger.log(MLevel.FINER, "Preparing to destroy resource: " + resc);
963 
964                     mgr.destroyResource(resc); 
965 
966                     if (Debug.DEBUG && Debug.TRACE == Debug.TRACE_MAX && logger.isLoggable( MLevel.FINER ))
967                         logger.log(MLevel.FINER, "Successfully destroyed resource: " + resc);
968                 }
969                 catch ( Exception   e )
970                 {
971                     if ( logger.isLoggable( MLevel.WARNING ) )
972                         logger.log( MLevel.WARNING, "Failed to destroy resource: " + resc, e );
973 
974                     // System.err.println("Failed to destroy resource: " + resc);
975                     // e.printStackTrace();
976                 }
977             }
978         }
979 
980         Runnable   r = new DestroyResourceTask();
981         if ( synchronous || broken ) //if we're broken, our taskRunner may be dead, so we destroy synchronously
982         {
983             if ( logger.isLoggable(MLevel.FINEST) && !broken && Boolean.TRUE.equals( ThreadUtils.reflectiveHoldsLock( this ) ) )
984                 logger.log( MLevel.FINEST, 
985                                 this + ": Destroyiong a resource on an active pool, synchronousy while holding pool's lock! " +
986                                 "(not a bug, but a potential bottleneck... is there a good reason for this?)", 
987                                 new Exception  ("DEBUG STACK TRACE") );
988 
989             r.run();
990         }
991         else
992         {
993             try { taskRunner.postRunnable( r ); }
994             catch (Exception   e)
995             {
996                 if (logger.isLoggable(MLevel.FINER))
997                     logger.log( MLevel.FINER, 
998                                     "AsynchronousRunner refused to accept task to destroy resource. " +
999                                     "It is probably shared, and has probably been closed underneath us. " +
1000                                    "Reverting to synchronous destruction. This is not usually a problem.",
1001                                    e );
1002                destroyResource( resc, true );
1003            }
1004        }
1005    }
1006
1007
1008    //this method SHOULD NOT be invoked from a synchronized
1009    //block!!!!
1010    private void doAcquire() throws Exception  
1011    {
1012        assert !Thread.holdsLock( this );
1013
1014        Object   resc = mgr.acquireResource(); //note we acquire the resource while we DO NOT hold the pool's lock!
1015
1016        boolean destroy = false;
1017        int msz;
1018
1019        synchronized(this) //assimilate resc if we do need it
1020        {
1021//          ++total_acquired;
1022
1023//          if (logger.isLoggable( MLevel.FINER))
1024//          logger.log(MLevel.FINER, "acquired new resource, total_acquired: " + total_acquired);
1025
1026            msz = managed.size();
1027            if (msz < target_pool_size)
1028                assimilateResource(resc); 
1029            else
1030                destroy = true;
1031        }
1032
1033        if (destroy)
1034        {
1035            mgr.destroyResource( resc ); //destroy resc if superfluous, without holding the pool's lock
1036            if (logger.isLoggable( MLevel.FINER))
1037                logger.log(MLevel.FINER, "destroying overacquired resource: " + resc);
1038        }
1039
1040    }
1041
1042    public synchronized void setPoolSize( int sz ) throws ResourcePoolException
1043    {
1044        try
1045        {
1046            setTargetPoolSize( sz );
1047            while ( managed.size() != sz )
1048                this.wait();
1049        }
1050        catch (Exception   e)
1051        {
1052            String   msg = "An exception occurred while trying to set the pool size!";
1053            if ( logger.isLoggable( MLevel.FINER ) )
1054                logger.log( MLevel.FINER, msg, e );
1055            throw ResourcePoolUtils.convertThrowable( msg, e );
1056        }
1057    }
1058
1059    public synchronized void setTargetPoolSize(int sz)
1060    {
1061        if (sz > max)
1062        {
1063            throw new IllegalArgumentException  ("Requested size [" + sz + 
1064                            "] is greater than max [" + max +
1065            "].");
1066        } 
1067        else if (sz < min)
1068        {
1069            throw new IllegalArgumentException  ("Requested size [" + sz + 
1070                            "] is less than min [" + min +
1071            "].");
1072        }
1073
1074        this.target_pool_size = sz;
1075
1076        _recheckResizePool();
1077    }
1078
1079
1080//  private void acquireUntil(int num) throws Exception
1081//  {
1082//  int msz = managed.size();
1083//  for (int i = msz; i < num; ++i)
1084//  assimilateResource();
1085//  }
1086
1087    //the following methods should only be invoked from 
1088    //sync'ed methods / blocks...
1089
1090//  private Object useUnusedButNotInIdleCheck()
1091//  {
1092//  for (Iterator ii = unused.iterator(); ii.hasNext(); )
1093//  {
1094//  Object maybeOut = ii.next();
1095//  if (! idleCheckResources.contains( maybeOut ))
1096//  {
1097//  ii.remove();
1098//  return maybeOut;
1099//  }
1100//  }
1101//  throw new RuntimeException("Internal Error -- the pool determined that it did have a resource available for checkout, but was unable to find one.");
1102//  }
1103
1104//  private int actuallyAvailable()
1105//  { return unused.size() - idleCheckResources.size(); }
1106
1107    // must own this' lock
1108    private void markBrokenNoEnsureMinResources(Object   resc) 
1109    {
1110        assert Thread.holdsLock( this );
1111
1112        try
1113        { 
1114            _markBroken( resc ); 
1115        }
1116        catch ( ResourceClosedException e ) // one of our async threads died
1117        {
1118            //e.printStackTrace();
1119            if ( logger.isLoggable( MLevel.SEVERE ) )
1120                logger.log( MLevel.SEVERE, "Apparent pool break.", e );
1121            this.unexpectedBreak();
1122        }
1123    }
1124
1125    // must own this' lock
1126    private void _markBroken( Object   resc )
1127    {
1128        assert Thread.holdsLock( this );
1129
1130        if ( unused.contains( resc ) )
1131            removeResource( resc ); 
1132        else
1133            excludeResource( resc );
1134    }
1135
1136    //DEBUG
1137    //Exception firstClose = null;
1138
1139    public synchronized void close( boolean close_checked_out_resources )
1140    {
1141        if (! broken ) //ignore repeated calls to close
1142        {
1143            //DEBUG
1144            //firstClose = new Exception("First close() -- debug stack trace [CRAIG]");
1145            //firstClose.printStackTrace();
1146
1147            this.broken = true;
1148            final Collection cleanupResources = ( close_checked_out_resources ? (Collection) cloneOfManaged().keySet() : (Collection) cloneOfUnused() );
1149            if ( cullTask != null )
1150                cullTask.cancel();
1151            if (idleRefurbishTask != null)
1152                idleRefurbishTask.cancel();
1153
1154            // we destroy resources asynchronously, but with a dedicated one-off Thread, rather than
1155            // our asynchronous runner, because our asynchrous runner may be shutting down. The
1156            // destruction is asynchrounous because destroying a resource might require the resource's
1157            // lock, and we already have the pool's lock. But client threads may well have the resource's
1158            // lock while they try to check-in to the pool. The async destruction of resources avoids
1159            // the possibility of deadlock.
1160
1161            managed.keySet().removeAll( cleanupResources );
1162            unused.removeAll( cleanupResources );
1163            Thread   resourceDestroyer = new Thread  ("Resource Destroyer in BasicResourcePool.close()")
1164            {
1165                public void run()
1166                {
1167                    for (Iterator ii = cleanupResources.iterator(); ii.hasNext();)
1168                    {
1169                        try
1170                        {
1171                            Object   resc = ii.next();
1172                            //System.err.println("Destroying resource... " + resc);
1173
1174                            destroyResource( resc, true );
1175                        }
1176                        catch (Exception   e)
1177                        {
1178                            if (Debug.DEBUG) 
1179                            {
1180                                //e.printStackTrace();
1181                                if ( logger.isLoggable( MLevel.FINE ) )
1182                                    logger.log( MLevel.FINE, "BasicResourcePool -- A resource couldn't be cleaned up on close()", e );
1183                            }
1184                        }
1185                    }
1186                }
1187            };
1188            resourceDestroyer.start();
1189
1190            for (Iterator ii = acquireWaiters.iterator(); ii.hasNext(); )
1191                ((Thread  ) ii.next()).interrupt();
1192            for (Iterator ii = otherWaiters.iterator(); ii.hasNext(); )
1193                ((Thread  ) ii.next()).interrupt();
1194            if (factory != null)
1195                factory.markBroken( this );
1196
1197            // System.err.println(this + " closed.");
1198        }
1199        else
1200        {
1201            if ( logger.isLoggable( MLevel.WARNING ) )
1202                logger.warning(this + " -- close() called multiple times.");
1203            //System.err.println(this + " -- close() called multiple times.");
1204
1205            //DEBUG
1206            //firstClose.printStackTrace();
1207            //new Exception("Repeat close() [CRAIG]").printStackTrace();
1208        }
1209    }
1210
1211    private void doCheckinManaged( final Object   resc ) throws ResourcePoolException
1212    {
1213        assert Thread.holdsLock( this );
1214
1215        if (unused.contains(resc))
1216        {
1217            if ( Debug.DEBUG )
1218                throw new ResourcePoolException("Tried to check-in an already checked-in resource: " + resc);
1219        }
1220        else if (broken)
1221            removeResource( resc, true ); //synchronous... if we're broken, async tasks might not work
1222        else
1223        {
1224            class RefurbishCheckinResourceTask implements Runnable  
1225            {
1226                public void run()
1227                {
1228                    boolean resc_okay = attemptRefurbishResourceOnCheckin( resc );
1229                    synchronized( BasicResourcePool.this )
1230                    {
1231                        PunchCard card = (PunchCard) managed.get( resc );
1232
1233                        if ( resc_okay && card != null) //we have to check that the resource is still in the pool
1234                        {
1235                            unused.add(0,  resc );
1236
1237                            card.last_checkin_time = System.currentTimeMillis();
1238                            card.checkout_time = -1;
1239                        }
1240                        else
1241                        {
1242                            if (card != null)
1243                                card.checkout_time = -1; //so we don't see this as still checked out and log an overdue cxn in removeResource()
1244
1245                            removeResource( resc );
1246                            ensureMinResources();
1247
1248                            if (card == null && logger.isLoggable( MLevel.FINE ))
1249                                logger.fine("Resource " + resc + " was removed from the pool during its refurbishment for checkin.");
1250                        }
1251
1252                        asyncFireResourceCheckedIn( resc, managed.size(), unused.size(), excluded.size() );
1253                        BasicResourcePool.this.notifyAll();
1254                    }
1255                }
1256            }
1257
1258            Runnable   doMe = new RefurbishCheckinResourceTask();
1259            taskRunner.postRunnable( doMe );
1260        }
1261    }
1262
1263    private void doCheckinExcluded( Object   resc )
1264    {
1265        assert Thread.holdsLock( this );
1266
1267        excluded.remove(resc);
1268        destroyResource(resc);
1269    }
1270
1271    /*
1272     * by the semantics of wait(), a timeout of zero means forever.
1273     */
1274    private void awaitAvailable(long timeout) throws InterruptedException  , TimeoutException, ResourcePoolException
1275    {
1276        assert Thread.holdsLock( this );
1277
1278        if (force_kill_acquires)
1279            throw new ResourcePoolException("A ResourcePool cannot acquire a new resource -- the factory or source appears to be down.");
1280
1281        Thread   t = Thread.currentThread();
1282        try
1283        {
1284            acquireWaiters.add( t );
1285
1286            int avail;
1287            long start = ( timeout > 0 ? System.currentTimeMillis() : -1);
1288            if (Debug.DEBUG && Debug.TRACE == Debug.TRACE_MAX)
1289            {
1290                if ( logger.isLoggable( MLevel.FINE ) )
1291                    logger.fine("awaitAvailable(): " + 
1292                                    (exampleResource != null ? 
1293                                                    exampleResource : 
1294                                    "[unknown]") );
1295                trace();
1296            }
1297            while ((avail = unused.size()) == 0) 
1298            {
1299                // the if case below can only occur when 1) a user attempts a
1300                // checkout which would provoke an acquire; 2) this
1301                // increments the pending acquires, so we go to the
1302                // wait below without provoking postAcquireMore(); 3)
1303                // the resources are acquired; 4) external management
1304                // of the pool (via for instance unpoolResource() 
1305                // depletes the newly acquired resources before we
1306                // regain this' monitor; 5) we fall into wait() with
1307                // no acquires being scheduled, and perhaps a managed.size()
1308                // of zero, leading to deadlock. This could only occur in
1309                // fairly pathological situations where the pool is being
1310                // externally forced to a very low (even zero) size, but 
1311                // since I've seen it, I've fixed it.
1312                if (pending_acquires == 0 && managed.size() < max)
1313                    _recheckResizePool();
1314
1315                this.wait(timeout);
1316                if (timeout > 0 && System.currentTimeMillis() - start > timeout)
1317                    throw new TimeoutException("A client timed out while waiting to acquire a resource from " + this + " -- timeout at awaitAvailable()");
1318                if (force_kill_acquires)
1319                    throw new CannotAcquireResourceException("A ResourcePool could not acquire a resource from its primary factory or source.");
1320                ensureNotBroken();
1321            }
1322        }
1323        finally
1324        {
1325            acquireWaiters.remove( t );
1326            if (acquireWaiters.size() == 0)
1327                this.notifyAll();
1328        }
1329    }
1330
1331    private void assimilateResource( Object   resc ) throws Exception  
1332    {
1333        assert Thread.holdsLock( this );
1334
1335        managed.put(resc, new PunchCard());
1336        unused.add(0, resc);
1337        //System.err.println("assimilate resource... unused: " + unused.size());
1338        asyncFireResourceAcquired( resc, managed.size(), unused.size(), excluded.size() );
1339        this.notifyAll();
1340        if (Debug.DEBUG && Debug.TRACE == Debug.TRACE_MAX) trace();
1341        if (Debug.DEBUG && exampleResource == null)
1342            exampleResource = resc;
1343    }
1344
1345    // should NOT be called from synchronized method
1346    private void synchronousRemoveArbitraryResource()
1347    { 
1348        assert !Thread.holdsLock( this );
1349
1350        Object   removeMe = null;
1351
1352        synchronized ( this )
1353        {
1354            if (unused.size() > 0)
1355            {
1356                removeMe = unused.get(0);
1357                managed.remove(removeMe);
1358                unused.remove(removeMe);
1359            }
1360            else
1361            {
1362                Set checkedOut = cloneOfManaged().keySet();
1363                if ( checkedOut.isEmpty() )
1364                {
1365                    unexpectedBreak();
1366                    logger.severe("A pool from which a resource is requested to be removed appears to have no managed resources?!");
1367                }
1368                else
1369                    excludeResource( checkedOut.iterator().next() );
1370            }
1371        }
1372
1373        if (removeMe != null)
1374            destroyResource( removeMe, true );
1375    }
1376
1377    private void removeResource(Object   resc)
1378    { removeResource( resc, false ); }
1379
1380    private void removeResource(Object   resc, boolean synchronous)
1381    {
1382        assert Thread.holdsLock( this );
1383
1384        PunchCard pc = (PunchCard) managed.remove(resc);
1385
1386        if (pc != null)
1387        {
1388            if ( pc.checkout_time > 0 && !broken) //this is a checked-out resource in an active pool, must be overdue if we are removing it
1389            {
1390                if (logger.isLoggable( MLevel.INFO ) )
1391                {
1392                    logger.info("A checked-out resource is overdue, and will be destroyed: " + resc);
1393                    if (pc.checkoutStackTraceException != null)
1394                    {
1395                        logger.log( MLevel.INFO,
1396                                        "Logging the stack trace by which the overdue resource was checked-out.",
1397                                        pc.checkoutStackTraceException );
1398                    }
1399                }
1400            }
1401        }
1402        else if ( logger.isLoggable( MLevel.FINE ) )
1403            logger.fine("Resource " + resc + " was removed twice. (Lotsa reasons a resource can be removed, sometimes simultaneously. It's okay)");
1404
1405        unused.remove(resc);
1406        destroyResource(resc, synchronous);
1407        addToFormerResources( resc );
1408        asyncFireResourceRemoved( resc, false, managed.size(), unused.size(), excluded.size() );
1409
1410        if (Debug.DEBUG && Debug.TRACE == Debug.TRACE_MAX) trace();
1411        //System.err.println("RESOURCE REMOVED!");
1412    }
1413
1414    //when we want to conceptually remove a checked
1415    //out resource from the pool
1416    private void excludeResource(Object   resc)
1417    {
1418        assert Thread.holdsLock( this );
1419
1420        managed.remove(resc);
1421        excluded.add(resc);
1422        if (Debug.DEBUG && unused.contains(resc) )
1423            throw new InternalError  ( "We should only \"exclude\" checked-out resources!" );
1424        asyncFireResourceRemoved( resc, true, managed.size(), unused.size(), excluded.size() );
1425    }
1426
1427    private void removeTowards( int new_sz )
1428    {
1429        assert Thread.holdsLock( this );
1430
1431        int num_to_remove = managed.size() - new_sz;
1432        int count = 0;
1433        for (Iterator ii = cloneOfUnused().iterator(); 
1434        ii.hasNext() && count < num_to_remove; 
1435        ++count)
1436        {
1437            Object   resc = ii.next();
1438            removeResource( resc );
1439        }
1440    }
1441
1442    private void cullExpired()
1443    {
1444        assert Thread.holdsLock( this );
1445
1446        if ( logger.isLoggable( MLevel.FINER ) )
1447            logger.log( MLevel.FINER, "BEGIN check for expired resources.  [" + this + "]");
1448
1449        // if we do not time-out checkedout resources, we only need to test unused resources
1450        Collection checkMe = ( destroy_unreturned_resc_time > 0 ? (Collection) cloneOfManaged().keySet() : cloneOfUnused() );
1451
1452        for ( Iterator ii = checkMe.iterator(); ii.hasNext(); )
1453        {
1454            Object   resc = ii.next();
1455            if ( shouldExpire( resc ) )
1456            {
1457                if ( logger.isLoggable( MLevel.FINER ) )
1458                    logger.log( MLevel.FINER, "Removing expired resource: " + resc + " [" + this + "]");
1459
1460                target_pool_size = Math.max( min, target_pool_size - 1 ); //expiring a resource resources the target size to match
1461
1462                removeResource( resc );
1463
1464                if (Debug.DEBUG && Debug.TRACE == Debug.TRACE_MAX) trace();
1465            }
1466        }
1467        if ( logger.isLoggable( MLevel.FINER ) )
1468            logger.log( MLevel.FINER, "FINISHED check for expired resources.  [" + this + "]");
1469        ensureMinResources();
1470    }
1471
1472    private void checkIdleResources()
1473    {
1474        assert Thread.holdsLock( this );
1475
1476        List u = cloneOfUnused();
1477        for ( Iterator ii = u.iterator(); ii.hasNext(); )
1478        {
1479            Object   resc = ii.next();
1480            if ( idleCheckResources.add( resc ) )
1481                taskRunner.postRunnable( new AsyncTestIdleResourceTask( resc ) );
1482        }
1483
1484        if (Debug.DEBUG && Debug.TRACE == Debug.TRACE_MAX) trace();
1485    }
1486
1487    private boolean shouldExpire( Object   resc )
1488    {
1489        assert Thread.holdsLock( this );
1490
1491        boolean expired = false;
1492
1493        PunchCard pc = (PunchCard) managed.get( resc );
1494
1495        // the resource has already been removed
1496        // we return true, because removing twice does no harm
1497        // (false should work as well, but true seems safer.
1498        //  we certainly don't want to do anything else with
1499        //  this resource.)
1500        if (pc == null) 
1501        {
1502            if ( logger.isLoggable( MLevel.FINE ) )
1503                logger.fine( "Resource " + resc + " was being tested for expiration, but has already been removed from the pool.");
1504            return true;
1505        }
1506
1507        long now = System.currentTimeMillis();
1508
1509        if (pc.checkout_time < 0) //resource is not checked out
1510        {
1511            long idle_age = now - pc.last_checkin_time;
1512            if (excess_max_idle_time > 0)
1513            {
1514                int msz = managed.size();
1515                expired = (msz > min && idle_age > excess_max_idle_time);
1516                if ( expired && logger.isLoggable( MLevel.FINER ) )
1517                    logger.log(MLevel.FINER, 
1518                                    "EXPIRED excess idle resource: " + resc + 
1519                                    " ---> idle_time: " + idle_age + 
1520                                    "; excess_max_idle_time: " + excess_max_idle_time +
1521                                    "; pool_size: " + msz +
1522                                    "; min_pool_size: " + min +
1523                                    " [" + this + "]");
1524            }
1525            if (!expired && max_idle_time > 0)
1526            {
1527                expired = idle_age > max_idle_time;
1528                if ( expired && logger.isLoggable( MLevel.FINER ) )
1529                    logger.log(MLevel.FINER, 
1530                                    "EXPIRED idle resource: " + resc + 
1531                                    " ---> idle_time: " + idle_age + 
1532                                    "; max_idle_time: " + max_idle_time +
1533                                    " [" + this + "]");
1534            }
1535            if (!expired && max_resource_age > 0)
1536            {
1537                long abs_age = now - pc.acquisition_time;
1538                expired = ( abs_age > max_resource_age );
1539
1540                if ( expired && logger.isLoggable( MLevel.FINER ) )
1541                    logger.log(MLevel.FINER, 
1542                                    "EXPIRED old resource: " + resc + 
1543                                    " ---> absolute_age: " + abs_age + 
1544                                    "; max_absolute_age: " + max_resource_age +
1545                                    " [" + this + "]");
1546            }
1547        }
1548        else //resource is checked out
1549        {
1550            long checkout_age = now - pc.checkout_time;
1551            expired = checkout_age > destroy_unreturned_resc_time;
1552        }
1553
1554        return expired; 
1555    }
1556
1557
1558//  private boolean resourcesInIdleCheck()
1559//  { return idleCheckresources.size() > 0; }
1560
1561//  private int countAvailable()
1562//  { return unused.size() - idleCheckResources.size(); }
1563
1564
1565    // we needn't hold this' lock
1566    private void ensureStartResources()
1567    { recheckResizePool(); }
1568
1569    // we needn't hold this' lock
1570    private void ensureMinResources()
1571    { recheckResizePool(); }
1572
1573    private boolean attemptRefurbishResourceOnCheckout( Object   resc )
1574    {
1575        assert !Thread.holdsLock( this );
1576
1577        try
1578        { 
1579            mgr.refurbishResourceOnCheckout(resc); 
1580            return true;
1581        }
1582        catch (Exception   e)
1583        {
1584            //uh oh... bad resource...
1585            if (Debug.DEBUG) 
1586            {
1587                //e.printStackTrace();
1588                if (logger.isLoggable( MLevel.FINE ))
1589                    logger.log( MLevel.FINE, "A resource could not be refurbished for checkout. [" + resc + ']', e );
1590            }
1591            synchronized (this)
1592            {
1593                ++failed_checkouts;
1594                setLastCheckoutFailure(e);
1595            }
1596            return false;
1597        }
1598    }
1599
1600    private boolean attemptRefurbishResourceOnCheckin( Object   resc )
1601    {
1602        assert !Thread.holdsLock( this );
1603
1604        try
1605        { 
1606            mgr.refurbishResourceOnCheckin(resc); 
1607            return true;
1608        }
1609        catch (Exception   e)
1610        {
1611            //uh oh... bad resource...
1612            if (Debug.DEBUG) 
1613            {
1614                //e.printStackTrace();
1615                if (logger.isLoggable( MLevel.FINE ))
1616                    logger.log( MLevel.FINE, "A resource could not be refurbished on checkin. [" + resc + ']', e );
1617            }
1618            synchronized (this)
1619            {
1620                ++failed_checkins;
1621                setLastCheckinFailure(e);
1622            }
1623            return false;
1624        }
1625    }
1626
1627    private void ensureNotBroken() throws ResourcePoolException
1628    {
1629        assert Thread.holdsLock( this );
1630
1631        if (broken) 
1632            throw new ResourcePoolException("Attempted to use a closed or broken resource pool");
1633    }
1634
1635    private void trace()
1636    {
1637        assert Thread.holdsLock( this );
1638
1639        if ( logger.isLoggable( MLevel.FINEST ) )
1640        {
1641            String   exampleResStr = ( exampleResource == null ?
1642                            "" :
1643                                " (e.g. " + exampleResource +")");
1644            logger.finest("trace " + this + " [managed: " + managed.size() + ", " +
1645                            "unused: " + unused.size() + ", excluded: " +
1646                            excluded.size() + ']' + exampleResStr );
1647        }
1648    }
1649
1650    private final HashMap cloneOfManaged()
1651    { 
1652        assert Thread.holdsLock( this );
1653
1654        return (HashMap) managed.clone(); 
1655    }
1656
1657    private final LinkedList cloneOfUnused()
1658    { 
1659        assert Thread.holdsLock( this );
1660
1661        return (LinkedList) unused.clone(); 
1662    }
1663
1664    private final HashSet cloneOfExcluded()
1665    { 
1666        assert Thread.holdsLock( this );
1667
1668        return (HashSet) excluded.clone(); 
1669    }
1670
1671    class ScatteredAcquireTask implements Runnable  
1672    {
1673        int attempts_remaining;
1674
1675        ScatteredAcquireTask()
1676        { this ( (num_acq_attempts >= 0 ? num_acq_attempts : -1) , true ); }
1677
1678        private ScatteredAcquireTask(int attempts_remaining, boolean first_attempt)
1679        { 
1680            this.attempts_remaining = attempts_remaining; 
1681            if (first_attempt)
1682            {
1683                incrementPendingAcquires();
1684                if (logger.isLoggable(MLevel.FINEST))
1685                    logger.finest("Starting acquisition series. Incremented pending_acquires [" + pending_acquires + "], " +
1686                                    " attempts_remaining: " + attempts_remaining);
1687            }
1688            else
1689            {
1690                if (logger.isLoggable(MLevel.FINEST))
1691                    logger.finest("Continuing acquisition series. pending_acquires [" + pending_acquires + "], " +
1692                                    " attempts_remaining: " + attempts_remaining);
1693            }
1694        }
1695
1696        public void run()
1697        {
1698            try
1699            {
1700                boolean fkap = isForceKillAcquiresPending();
1701                if (! fkap)
1702                {
1703                    //we don't want this call to be sync'd
1704                    //on the pool, so that resource acquisition
1705                    //does not interfere with other pool clients.
1706                    BasicResourcePool.this.doAcquire();
1707                }
1708                decrementPendingAcquires();
1709                if (logger.isLoggable(MLevel.FINEST))
1710                    logger.finest("Acquisition series terminated " +
1711                                    (fkap ? "because force-kill-acquires is pending" : "successfully") +
1712                                    ". Decremented pending_acquires [" + pending_acquires + "], " +
1713                                    " attempts_remaining: " + attempts_remaining);
1714            }
1715            catch (Exception   e)
1716            {
1717                BasicResourcePool.this.setLastAcquisitionFailure(e);
1718
1719                if (attempts_remaining == 0) //last try in a round...
1720                {
1721                    decrementPendingAcquires();
1722                    if ( logger.isLoggable( MLevel.WARNING ) )
1723                    {
1724                        logger.log( MLevel.WARNING,
1725                                        this + " -- Acquisition Attempt Failed!!! Clearing pending acquires. " +
1726                                        "While trying to acquire a needed new resource, we failed " +
1727                                        "to succeed more than the maximum number of allowed " +
1728                                        "acquisition attempts (" + num_acq_attempts + "). " + 
1729                                        "Last acquisition attempt exception: ",
1730                                        e);
1731                    }
1732                    if (break_on_acquisition_failure)
1733                    {
1734                        //System.err.println("\tTHE RESOURCE POOL IS PERMANENTLY BROKEN!");
1735                        if ( logger.isLoggable( MLevel.SEVERE ) )
1736                            logger.severe("A RESOURCE POOL IS PERMANENTLY BROKEN! [" + this + "] " +
1737                                            "(because a series of " + num_acq_attempts + " acquisition attempts " +
1738                            "failed.)");
1739                        unexpectedBreak();
1740                    }
1741                    else
1742                    {
1743                        try { forceKillAcquires(); }
1744                        catch (InterruptedException   ie)
1745                        {
1746                            if ( logger.isLoggable(MLevel.WARNING) )
1747                                logger.log(MLevel.WARNING, 
1748                                                "Failed to force-kill pending acquisition attempts after acquisition failue, " +
1749                                                " due to an InterruptedException!",
1750                                                ie );
1751
1752                            // we might still have clients waiting, so we should try
1753                            // to ensure there are sufficient connections to serve
1754                            recheckResizePool();
1755                        }
1756                    }
1757                    if (logger.isLoggable(MLevel.FINEST))
1758                        logger.finest("Acquisition series terminated unsuccessfully. Decremented pending_acquires [" + pending_acquires + "], " +
1759                                        " attempts_remaining: " + attempts_remaining);
1760                }
1761                else
1762                {
1763                    // if attempts_remaining < 0, we try to acquire forever, so the end-of-batch
1764                    // log message below will never be triggered if there is a persistent problem
1765                    // so in this case, it's better flag a higher-than-debug-level message for
1766                    // each failed attempt. (Thanks to Eric Crahen for calling attention to this
1767                    // issue.)
1768                    MLevel logLevel = (attempts_remaining > 0 ? MLevel.FINE : MLevel.INFO);
1769                    if (logger.isLoggable( logLevel ))
1770                        logger.log( logLevel, "An exception occurred while acquiring a poolable resource. Will retry.", e );
1771
1772                    TimerTask doNextAcquire = new TimerTask()
1773                    {
1774                        public void run()
1775                        { taskRunner.postRunnable( new ScatteredAcquireTask( attempts_remaining - 1, false ) ); }
1776                    };
1777                    cullAndIdleRefurbishTimer.schedule( doNextAcquire, acq_attempt_delay );
1778                }
1779            }
1780        }
1781
1782    }
1783
1784    /*
1785     *  task we post to separate thread to acquire
1786     *  pooled resources
1787     */
1788    class AcquireTask implements Runnable  
1789    {
1790        boolean success = false;
1791
1792        public AcquireTask() 
1793        { incrementPendingAcquires(); }
1794
1795        public void run()
1796        {
1797            try
1798            {
1799                Exception   lastException = null;
1800                for (int i = 0; shouldTry( i ); ++i)
1801                {
1802                    try
1803                    {
1804                        if (i > 0)
1805                            Thread.sleep(acq_attempt_delay); 
1806
1807                        //we don't want this call to be sync'd
1808                        //on the pool, so that resource acquisition
1809                        //does not interfere with other pool clients.
1810                        BasicResourcePool.this.doAcquire();
1811
1812                        success = true;
1813                    }
1814                    catch (InterruptedException   e)
1815                    {
1816                        // end the whole task on interrupt, regardless of success
1817                        // or failure
1818                        throw e;
1819                    }
1820                    catch (Exception   e)
1821                    {
1822                        //e.printStackTrace();
1823
1824                        // if num_acq_attempts <= 0, we try to acquire forever, so the end-of-batch
1825                        // log message below will never be triggered if there is a persistent problem
1826                        // so in this case, it's better flag a higher-than-debug-level message for
1827                        // each failed attempt. (Thanks to Eric Crahen for calling attention to this
1828                        // issue.)
1829                        MLevel logLevel = (num_acq_attempts > 0 ? MLevel.FINE : MLevel.INFO);
1830                        if (logger.isLoggable( logLevel ))
1831                            logger.log( logLevel, "An exception occurred while acquiring a poolable resource. Will retry.", e );
1832
1833                        lastException = e;
1834                        setLastAcquisitionFailure(e);
1835                    }
1836                }
1837                if (!success) 
1838                {
1839                    if ( logger.isLoggable( MLevel.WARNING ) )
1840                    {
1841                        logger.log( MLevel.WARNING,
1842                                        this + " -- Acquisition Attempt Failed!!! Clearing pending acquires. " +
1843                                        "While trying to acquire a needed new resource, we failed " +
1844                                        "to succeed more than the maximum number of allowed " +
1845                                        "acquisition attempts (" + num_acq_attempts + "). " + 
1846                                        (lastException == null ? "" : "Last acquisition attempt exception: "),
1847                                        lastException);
1848                    }
1849                    if (break_on_acquisition_failure)
1850                    {
1851                        //System.err.println("\tTHE RESOURCE POOL IS PERMANENTLY BROKEN!");
1852                        if ( logger.isLoggable( MLevel.SEVERE ) )
1853                            logger.severe("A RESOURCE POOL IS PERMANENTLY BROKEN! [" + this + "]");
1854                        unexpectedBreak();
1855                    }
1856                    else
1857                        forceKillAcquires();
1858                }
1859                else
1860                    recheckResizePool();
1861            }
1862            catch ( ResourceClosedException e ) // one of our async threads died
1863            {
1864                //e.printStackTrace();
1865                if ( Debug.DEBUG )
1866                {
1867                    if ( logger.isLoggable( MLevel.FINE ) )
1868                        logger.log( MLevel.FINE, "a resource pool async thread died.", e );
1869                }
1870                unexpectedBreak();
1871            }
1872            catch (InterruptedException   e) //from force kill acquires, or by the thread pool during the long task...
1873            {
1874                if ( logger.isLoggable( MLevel.WARNING ) )
1875                {
1876                    logger.log( MLevel.WARNING,
1877                                    BasicResourcePool.this + " -- Thread unexpectedly interrupted while performing an acquisition attempt.",
1878                                    e );
1879                }
1880
1881//              System.err.println(BasicResourcePool.this + " -- Thread unexpectedly interrupted while waiting for stale acquisition attempts to die.");
1882//              e.printStackTrace();
1883
1884                recheckResizePool();
1885            }
1886            finally
1887            { decrementPendingAcquires(); }
1888        }
1889
1890        private boolean shouldTry(int attempt_num)
1891        {
1892            //try if we haven't already succeeded
1893            //and someone hasn't signalled that our resource source is down
1894            //and not max attempts is set,
1895            //or we are less than the set limit
1896            return 
1897            !success && 
1898            !isForceKillAcquiresPending() &&
1899            (num_acq_attempts <= 0 || attempt_num < num_acq_attempts);
1900        }
1901    }
1902
1903    /*
1904     *  task we post to separate thread to remove
1905     *  unspecified pooled resources
1906     *
1907     *  TODO: do removal and destruction synchronously
1908     *        but carefully not synchronized during the
1909     *        destruction of the resource.
1910     */
1911    class RemoveTask implements Runnable  
1912    {
1913        public RemoveTask() 
1914        { incrementPendingRemoves(); }
1915
1916        public void run()
1917        {
1918            try
1919            {
1920                synchronousRemoveArbitraryResource();
1921                recheckResizePool();
1922            }
1923            finally
1924            { decrementPendingRemoves(); }
1925        }
1926    }
1927
1928    class CullTask extends TimerTask
1929    {
1930        public void run()
1931        {
1932            try
1933            {
1934                if (Debug.DEBUG && Debug.TRACE >= Debug.TRACE_MED && logger.isLoggable( MLevel.FINER ))
1935                    logger.log( MLevel.FINER, "Checking for expired resources - " + new Date() + " [" + BasicResourcePool.this + "]");
1936                synchronized ( BasicResourcePool.this )
1937                { cullExpired(); }
1938            }
1939            catch ( ResourceClosedException e ) // one of our async threads died
1940            {
1941                if ( Debug.DEBUG )
1942                {
1943                    if ( logger.isLoggable( MLevel.FINE ) )
1944                        logger.log( MLevel.FINE, "a resource pool async thread died.", e );
1945                }
1946                unexpectedBreak();
1947            }
1948        }
1949    }
1950
1951    // this is run by a single-threaded timer, so we don't have
1952    // to worry about multiple threads executing the task at the same 
1953    // time 
1954    class CheckIdleResourcesTask extends TimerTask
1955    {
1956        public void run()
1957        {
1958            try
1959            {
1960                //System.err.println("c3p0-JENNIFER: refurbishing idle resources - " + new Date() + " [" + BasicResourcePool.this + "]");
1961                if (Debug.DEBUG && Debug.TRACE >= Debug.TRACE_MED && logger.isLoggable(MLevel.FINER))
1962                    logger.log(MLevel.FINER, "Refurbishing idle resources - " + new Date() + " [" + BasicResourcePool.this + "]");
1963                synchronized ( BasicResourcePool.this )
1964                { checkIdleResources(); }
1965            }
1966            catch ( ResourceClosedException e ) // one of our async threads died
1967            {
1968                //e.printStackTrace();
1969                if ( Debug.DEBUG )
1970                {
1971                    if ( logger.isLoggable( MLevel.FINE ) )
1972                        logger.log( MLevel.FINE, "a resource pool async thread died.", e );
1973                }
1974                unexpectedBreak();
1975            }
1976        }
1977    }
1978
1979    class AsyncTestIdleResourceTask implements Runnable  
1980    {
1981        // unchanging after ctor
1982        Object   resc;
1983
1984        // protected by this' lock
1985        boolean pending = true;
1986        boolean failed;
1987
1988        AsyncTestIdleResourceTask( Object   resc )
1989        { this.resc = resc; }
1990
1991        public void run()
1992        {
1993            assert !Thread.holdsLock( BasicResourcePool.this );
1994
1995            try
1996            {
1997                try
1998                { 
1999                    mgr.refurbishIdleResource( resc ); 
2000                }
2001                catch ( Exception   e )
2002                {
2003                    if ( logger.isLoggable( MLevel.FINE ) )
2004                        logger.log( MLevel.FINE, "BasicResourcePool: An idle resource is broken and will be purged. [" + resc + ']', e);
2005
2006                    synchronized (BasicResourcePool.this)
2007                    {
2008                        if ( managed.keySet().contains( resc ) ) //resc might have been culled as expired while we tested
2009                        {
2010                            removeResource( resc ); 
2011                            ensureMinResources();
2012                        }
2013
2014                        ++failed_idle_tests;
2015                        setLastIdleCheckFailure(e);
2016                    }
2017                }
2018            }
2019            finally
2020            {
2021                synchronized (BasicResourcePool.this)
2022                {
2023                    idleCheckResources.remove( resc );
2024                    BasicResourcePool.this.notifyAll();
2025                }
2026            }
2027        }
2028    }
2029
2030    final static class PunchCard
2031    {
2032        long acquisition_time;
2033        long last_checkin_time;
2034        long checkout_time;
2035        Exception   checkoutStackTraceException;
2036
2037        PunchCard()
2038        {
2039            this.acquisition_time = System.currentTimeMillis();
2040            this.last_checkin_time = acquisition_time;
2041            this.checkout_time = -1;
2042            this.checkoutStackTraceException = null;
2043        }
2044    }
2045
2046//  static class CheckInProgressResourceHolder
2047//  {
2048//  Object checkResource;
2049
2050//  public synchronized void setCheckResource( Object resc )
2051//  { 
2052//  this.checkResource = resc; 
2053//  this.notifyAll();
2054//  }
2055
2056//  public void unsetCheckResource()
2057//  { setCheckResource( null ); }
2058
2059//  /**
2060//  * @return true if we actually had to wait
2061//  */
2062//  public synchronized boolean awaitNotInCheck( Object resc )
2063//  {
2064//  boolean had_to_wait = false;
2065//  boolean set_interrupt = false;
2066//  while ( checkResource == resc )
2067//  {
2068//  try
2069//  {
2070//  had_to_wait = true;
2071//  this.wait(); 
2072//  }
2073//  catch ( InterruptedException e )
2074//  { 
2075//  e.printStackTrace();
2076//  set_interrupt = true;
2077//  }
2078//  }
2079//  if ( set_interrupt )
2080//  Thread.currentThread().interrupt();
2081//  return had_to_wait;
2082//  }
2083//  }
2084}
2085
2086
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags