WriterPool


1   /* WriterPool
2    *
3    * $Id: WriterPool.java,v 1.4 2006/08/25 00:14:29 stack-sf Exp $
4    *
5    * Created July 19th, 2006.
6    *
7    * Copyright (C) 2006 Internet Archive.
8    *
9    * This file is part of the Heritrix web crawler (crawler.archive.org).
10   *
11   * Heritrix is free software; you can redistribute it and/or modify
12   * it under the terms of the GNU Lesser Public License as published by
13   * the Free Software Foundation; either version 2.1 of the License, or
14   * any later version.
15   *
16   * Heritrix is distributed in the hope that it will be useful,
17   * but WITHOUT ANY WARRANTY; without even the implied warranty of
18   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19   * GNU Lesser Public License for more details.
20   *
21   * You should have received a copy of the GNU Lesser Public License
22   * along with Heritrix; if not, write to the Free Software
23   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
24   */
25  package org.archive.io;
26  
27  import java.io.File  ;
28  import java.io.IOException  ;
29  import java.util.NoSuchElementException  ;
30  import java.util.concurrent.atomic.AtomicInteger  ;
31  import java.util.logging.Level  ;
32  import java.util.logging.Logger  ;
33  
34  import org.apache.commons.pool.BasePoolableObjectFactory;
35  import org.apache.commons.pool.impl.FairGenericObjectPool;
36  import org.apache.commons.pool.impl.GenericObjectPool;
37  
38  /**
39   * Pool of Writers.
40   * 
41   * Abstract. Override and pass in the Constructor a factory that creates
42   * {@link WriterPoolMember} implementations.
43   * 
44   * @author stack
45   */
46  public abstract class WriterPool {
47      final Logger   logger =  Logger.getLogger(this.getClass().getName());
48     
49      /**
50       * Used to generate unique filename sequences.
51       */
52      final private AtomicInteger   serialNo;
53      
54      /**
55       * Don't enforce a maximum number of idle instances in pool.
56       * To do so means GenericObjectPool will close files prematurely.
57       */
58      protected static final int NO_MAX_IDLE = -1;
59      
60      /**
61       * Retry getting a file on fail the below arbitrary amount of times.
62       * This facility is not configurable.  If we fail this many times
63       * getting a file, something is seriously wrong.
64       */
65      private final int arbitraryRetryMax = 10;
66      
67      /**
68       * Default maximum active number of files in the pool.
69       */
70      public static final int DEFAULT_MAX_ACTIVE = 5;
71  
72      /**
73       * Maximum time to wait on a free file..
74       */
75      public static final int DEFAULT_MAXIMUM_WAIT = 1000 * 60 * 5;
76      
77      /**
78       * Pool instance.
79       */
80      private GenericObjectPool pool = null;
81      
82      /**
83       * File settings.
84       * Keep in data structure rather than as individual values.
85       */
86      private final WriterPoolSettings settings;
87      
88      /**
89       * Shutdown default constructor.
90       */
91      private WriterPool() {
92          this(null, null, null, -1, -1);
93      }
94      
95      /**
96       * Constructor
97       * @param serial  Used to generate unique filename sequences
98       * @param factory Factory that knows how to make a {@link WriterPoolMember}.
99       * @param settings Settings for this pool.
100      * @param poolMaximumActive
101      * @param poolMaximumWait
102      */
103     public WriterPool(final AtomicInteger   serial,
104             final BasePoolableObjectFactory factory,
105             final WriterPoolSettings settings,
106             final int poolMaximumActive, final int poolMaximumWait) {
107         logger.info("Initial configuration:" +
108                 " prefix=" + settings.getPrefix() +
109                 ", suffix=" + settings.getSuffix() +
110                 ", compress=" + settings.isCompressed() +
111                 ", maxSize=" + settings.getMaxSize() +
112                 ", maxActive=" + poolMaximumActive +
113                 ", maxWait=" + poolMaximumWait);
114         this.settings = settings;
115         this.pool = new FairGenericObjectPool(factory, poolMaximumActive,
116             GenericObjectPool.WHEN_EXHAUSTED_BLOCK, poolMaximumWait,
117             NO_MAX_IDLE);
118         this.serialNo = serial;
119     }
120 
121     /**
122      * Check out a {@link WriterPoolMember}.
123      * 
124      * This method must be answered by a call to
125      * {@link #returnFile(WriterPoolMember)} else pool starts leaking.
126      * 
127      * @return Writer checked out of a pool of files.
128      * @throws IOException Problem getting Writer from pool (Converted
129      * from Exception to IOException so this pool can live as a good citizen
130      * down in depths of ARCSocketFactory).
131      * @throws NoSuchElementException If we time out waiting on a pool member.
132      */
133     public WriterPoolMember borrowFile()
134     throws IOException   {
135         WriterPoolMember f = null;
136         for (int i = 0; f == null; i++) {
137             long waitStart = System.currentTimeMillis();
138             try {
139                 f = (WriterPoolMember)this.pool.borrowObject();
140                 if (logger.getLevel() == Level.FINE) {
141                     logger.fine("Borrowed " + f + " (Pool State: "
142                         + getPoolState(waitStart) + ").");
143                 }
144             } catch (NoSuchElementException   e) {
145                 // Let this exception out. Unit test at least depends on it.
146                 // Log current state of the pool.
147                 logger.warning(e.getMessage() + ": Retry #" + i + " of "
148                     + " max of " + arbitraryRetryMax
149                     + ": NSEE Pool State: " + getPoolState(waitStart));
150                 if (i >= arbitraryRetryMax) {
151                     logger.log(Level.SEVERE,
152                         "maximum retries exceeded; rethrowing",e);
153                     throw e;
154                 }
155             } catch (Exception   e) {
156                 // Convert.
157                 logger.severe(e.getMessage() + ": E Pool State: " +
158                     getPoolState(waitStart));
159                 throw new IOException  ("Failed getting writer from pool: " +
160                     e.getMessage());
161             }
162         }
163         return f;
164     }
165 
166     /**
167      * @param writer Writer to return to the pool.
168      * @throws IOException Problem returning File to pool.
169      */
170     public void returnFile(WriterPoolMember writer)
171     throws IOException   {
172         try {
173             if (logger.getLevel() == Level.FINE) {
174                 logger.fine("Returned " + writer);
175             }
176             this.pool.returnObject(writer);
177         }
178         catch(Exception   e)
179         {
180             throw new IOException  ("Failed restoring writer to pool: " +
181                     e.getMessage());
182         }
183     }
184 
185     public void invalidateFile(WriterPoolMember f)
186     throws IOException   {
187         try {
188             this.pool.invalidateObject(f);
189         } catch (Exception   e) {
190             // Convert exception.
191             throw new IOException  (e.getMessage());
192         }
193         // It'll have been closed.  Rename with an '.invalid' suffix so it
194         // gets attention.
195         File   file = f.getFile();
196         file.renameTo(new File  (file.getAbsoluteFile() +
197                 WriterPoolMember.INVALID_SUFFIX));
198     }
199 
200     /**
201      * @return Number of {@link WriterPoolMember}s checked out of pool.
202      * @throws java.lang.UnsupportedOperationException
203      */
204     public int getNumActive()
205     throws UnsupportedOperationException   {
206         return this.pool.getNumActive();
207     }
208 
209     /**
210      * @return Number of {@link WriterPoolMember} instances still in the pool.
211      * @throws java.lang.UnsupportedOperationException
212      */
213     public int getNumIdle()
214     throws UnsupportedOperationException   {
215         return this.pool.getNumIdle();
216     }
217     
218     /**
219      * Close all {@link WriterPoolMember}s in pool.
220      */
221     public void close() {
222         this.pool.clear();
223     }
224 
225     /**
226      * @return Returns settings.
227      */
228     public WriterPoolSettings getSettings() {
229         return this.settings;
230     }
231     
232     /**
233      * @return State of the pool string
234      */
235     protected String   getPoolState() {
236         return getPoolState(-1);
237     }
238     
239     /**
240      * @param startTime If we are passed a start time, we'll add difference
241      * between it and now to end of string.  Pass -1 if don't want this
242      * added to end of state string.
243      * @return State of the pool string
244      */
245     protected String   getPoolState(long startTime) {
246         StringBuffer   buffer = new StringBuffer  ("Active ");
247         buffer.append(getNumActive());
248         buffer.append(" of max ");
249         buffer.append(this.pool.getMaxActive());
250         buffer.append(", idle ");
251         buffer.append(this.pool.getNumIdle());
252         if (startTime != -1) {
253             buffer.append(", time ");
254             buffer.append(System.currentTimeMillis() - startTime);
255             buffer.append("ms of max ");
256             buffer.append(this.pool.getMaxWait());
257             buffer.append("ms");
258         }
259         return buffer.toString();
260     }
261     
262     /**
263      * Returns the atomic integer used to generate serial numbers
264      * for files.
265      * 
266      * @return  the serial number generator
267      */
268     public AtomicInteger   getSerialNo() {
269         return serialNo;
270     }
271 }
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags