Java > Open Source Codes > org > jboss > cache > loader > deadlock > ConcurrentCreationDeadlockTest


1 package org.jboss.cache.loader.deadlock;
2
3 import EDU.oswego.cs.dl.util.concurrent.Latch;
4 import junit.framework.Test;
5 import junit.framework.TestSuite;
6 import org.jboss.cache.CacheImpl;
7 import org.jboss.cache.Fqn;
8 import org.jboss.cache.config.Configuration;
9 import org.jboss.cache.factories.XmlConfigurationParser;
10 import org.jboss.cache.loader.AbstractCacheLoaderTestBase;
11 import org.jboss.cache.transaction.DummyTransactionManager;
12
13 import javax.naming.Context ;
14 import javax.naming.InitialContext ;
15 import javax.transaction.UserTransaction ;
16 import java.util.Properties ;
17
18 /**
19  * <b>Test based on a contribution by Marian Nokolov/Paul Miodonski at Siemens AG.</b>
20  * <p/>
21  * This test has been created to simulate a unexpected TimeoutException that is
22  * thrown by the JBossCache. The original scenario that has been observed: <br>
23  * Cache in either LOCAL or REPL_SYNC mode with CacheLoader.
24  * <ul>
25  * <li>1. Concurrent threads A, B and C, each associated with a transaction.
26  * Threads A and B try to modify FQN X, thread C tries to modify FQN Y.</li>
27  * <li>2. Thread A locks X.</li>
28  * <li>3. Thread B blocks on X (correct since A is the active writer).</li>
29  * <li>4. Thread A tries to do multiple modifications and suddenly blocks on X
30  * (although it is the active writer already) - this is the 1-st problem.</li>
31  * <li>5. Thread C blocks somewhere as well, although it has nothing to do with
32  * X and is the only one that works on Y - this is a 2-nd problem.</li>
33  * <li>6. Thread B fails with TimeoutException and its transaction is rolled
34  * back - this is correct given that A is still holding the lock on X.</li>
35  * <li>7. Thread A continues its job and its transaction completes successfully
36  * (after unexpected locking timeout delay).</li>
37  * <li>8. Thread C continues its job and successfully commits the transaction
38  * (with unexpected locking timeout delay).</li>
39  * </ul>
40  * <br>
41  * There are two problems with this:
42  * <ul>
43  * <li>1. One or more concurrent transactions fail, although the pessimistic
44  * locking should sequentialize them and guarantee that all should succeed.</li>
45  * <li>2. Any other thread that tries to acquire a lock in the cache (even for
46  * a different FQN!?) is blocked for the duration of the locking timeout, i.e.
47  * the whole application is locked for few seconds. The active writer for the
48  * corresponding FQN is blocked as well in the middle of the transaction!</li>
49  * </ul>
50  * <br>
51  * At least until now, the error can be reproduced only if the following is
52  * true:
53  * <ul>
54  * <li>Concurrent transactions forcing creation of the same FQN at the same
55  * time.</li>
56  * <li>More than one update per TX per FQN - trying to acquire lock on the same
57  * FQN multiple times per TX.</li>
58  * <li>Cache with CacheLoader - maybe it has something to do with the
59  * CacheLoader/StoreInterceptor's...</li>
60  * </ul>
61  */
62 public class ConcurrentCreationDeadlockTest extends AbstractCacheLoaderTestBase
63 {
64    /**
65     * The number of worker threads to start concurrently.
66     */
67    private static final int NUM_WORKERS = 10;
68    /**
69     * The number of test runs to perform.
70     */
71    private static final int NUM_RUNS = 100;
72    /**
73     * The number of FQN's per test run.
74     */
75    private static final int NUM_FQNS_PER_RUN = 10;
76
77    /**
78     * The initial context factory properties.
79     */
80    private static final Properties PROPERTIES;
81    /**
82     * The context factory to be used for the test.
83     */
84    private static final String CONTEXT_FACTORY =
85            "org.jboss.cache.transaction.DummyContextFactory";
86    /**
87     * The original context factory to be restored after the test.
88     */
89    private String m_contextFactory = null;
90
91    /**
92     * Exception recorded if any of the worker threads fails.
93     */
94    private static volatile Exception mcl_exception = null;
95
96    /**
97     * The cache under test.
98     */
99    private CacheImpl cache = null;
100
101    static
102    {
103       PROPERTIES = new Properties ();
104       PROPERTIES.put(Context.INITIAL_CONTEXT_FACTORY,
105               "org.jboss.cache.transaction.DummyContextFactory");
106    }
107
108    /**
109     * Constructor.
110     *
111     * @param name The test name.
112     */
113    public ConcurrentCreationDeadlockTest(String name)
114    {
115       super(name);
116    }
117
118    /**
119     * {@inheritDoc}
120     */
121    public void setUp() throws Exception
122    {
123       super.setUp();
124       mcl_exception = null;
125       m_contextFactory = System.getProperty(Context.INITIAL_CONTEXT_FACTORY);
126       System.setProperty(Context.INITIAL_CONTEXT_FACTORY, CONTEXT_FACTORY);
127       DummyTransactionManager.getInstance();
128       cache = new CacheImpl();
129
130       XmlConfigurationParser parser = new XmlConfigurationParser();
131       Configuration c = parser.parseFile("META-INF/replSync-service.xml");
132       cache.setConfiguration(c);
133
134       c.setTransactionManagerLookupClass("org.jboss.cache.DummyTransactionManagerLookup");
135    }
136
137    /**
138     * {@inheritDoc}
139     */
140    public void tearDown() throws Exception
141    {
142       super.tearDown();
143       DummyTransactionManager.destroy();
144       cache.stop();
145       cache = null;
146       if (m_contextFactory != null)
147       {
148          System.setProperty(Context.INITIAL_CONTEXT_FACTORY,
149                  m_contextFactory);
150          m_contextFactory = null;
151       }
152    }
153
154    /**
155     * Initializes and starts the cache.
156     *
157     * @param cacheMode The cache mode.
158     * @param cacheLoaderClass The name of the cache loader class.
159     * @throws Exception Any exception if thrown by the cache.
160     */
161    private void startCache(Configuration.CacheMode cacheMode, String cacheLoaderClass)
162            throws Exception
163    {
164       cache.getConfiguration().setCacheMode(cacheMode);
165       if (cacheLoaderClass != null)
166       {
167          cache.getConfiguration().setCacheLoaderConfig(getSingleCacheLoaderConfig("", cacheLoaderClass, "", false, false, false));
168       }
169       cache.create();
170       cache.start();
171    }
172
173    /**
174     * Test for a local cache without cache loader and single modification per
175     * transaction.<br>
176     * This test has never failed up to now.
177     *
178     * @throws Exception Any exception if thrown by the cache.
179     */
180    public void disabledtestLocalNoCacheLoader1Modification() throws Exception
181    {
182       startCache(Configuration.CacheMode.LOCAL, null);
183       performTest(1);
184    }
185
186    /**
187     * Test for a local cache without cache loader and two modifications per
188     * transaction.<br>
189     * This test has never failed up to now.
190     *
191     * @throws Exception Any exception if thrown by the cache.
192     */
193    public void disabledtestLocalNoCacheLoader2Modifications()
194            throws Exception
195    {
196       startCache(Configuration.CacheMode.LOCAL, null);
197       performTest(2);
198    }
199
200    /**
201     * Test for a local cache with cache loader and single modification per
202     * transaction.<br>
203     * This test has never failed up to now.
204     *
205     * @throws Exception Any exception if thrown by the cache.
206     */
207    public void disabledtestLocalCacheLoader1Modification() throws Exception
208    {
209       startCache(Configuration.CacheMode.LOCAL, "org.jboss.cache.loader.DummyCacheLoader");
210       performTest(1);
211    }
212
213    /**
214     * Test for a local cache with cache loader and two modifications per
215     * transaction.<br>
216     * This test does very often fail with a TimeoutException.
217     *
218     * @throws Exception Any exception if thrown by the cache.
219     */
220    public void testLocalCacheLoader2Modifications() throws Exception
221    {
222       startCache(Configuration.CacheMode.LOCAL, "org.jboss.cache.loader.DummyCacheLoader");
223       performTest(2);
224    }
225
226    /**
227     * Test for an asynchronously replicated cache without cache loader and
228     * single modification per transaction.<br>
229     * This test has never failed up to now.
230     *
231     * @throws Exception Any exception if thrown by the cache.
232     */
233    public void disabledtestReplAsyncNoCacheLoader1Modification()
234            throws Exception
235    {
236       startCache(Configuration.CacheMode.REPL_ASYNC, null);
237       performTest(1);
238    }
239
240    /**
241     * Test for an asynchronously replicated cache without cache loader and two
242     * modifications per transaction.<br>
243     * This test has never failed up to now.
244     *
245     * @throws Exception Any exception if thrown by the cache.
246     */
247    public void disabledtestReplAsyncNoCacheLoader2Modifications()
248            throws Exception
249    {
250       startCache(Configuration.CacheMode.REPL_ASYNC, null);
251       performTest(2);
252    }
253
254    /**
255     * Test for an asynchronously replicated cache with cache loader and single
256     * modification per transaction.<br>
257     * This test has never failed up to now.
258     *
259     * @throws Exception Any exception if thrown by the cache.
260     */
261    public void disabledtestReplAsyncCacheLoader1Modification()
262            throws Exception
263    {
264       startCache(Configuration.CacheMode.REPL_ASYNC, "org.jboss.cache.loader.DummyCacheLoader");
265       performTest(1);
266    }
267
268    /**
269     * Test for an asynchronously replicated cache with cache loader and two
270     * modification per transaction.<br>
271     * This test mysteriously never fails, although it should as the LOCAL
272     * and REPL_SYNC do.
273     *
274     * @throws Exception Any exception if thrown by the cache.
275     */
276    public void disabledtestReplAsyncCacheLoader2Modifications()
277            throws Exception
278    {
279       startCache(Configuration.CacheMode.REPL_ASYNC, "org.jboss.cache.loader.DummyCacheLoader");
280       performTest(2);
281    }
282
283    /**
284     * Test for a synchronously replicated cache without cache loader and single
285     * modification per transaction.<br>
286     * This test has never failed up to now.
287     *
288     * @throws Exception Any exception if thrown by the cache.
289     */
290    public void disabledtestReplSyncNoCacheLoader1Modification()
291            throws Exception
292    {
293       startCache(Configuration.CacheMode.REPL_SYNC, null);
294       performTest(1);
295    }
296
297    /**
298     * Test for a synchronously replicated cache without cache loader and two
299     * modification per transaction.<br>
300     * This test has never failed up to now.
301     *
302     * @throws Exception Any exception if thrown by the cache.
303     */
304    public void disabledtestReplSyncNoCacheLoader2Modifications()
305            throws Exception
306    {
307       startCache(Configuration.CacheMode.REPL_SYNC, null);
308       performTest(2);
309    }
310
311    /**
312     * Test for a synchronously replicated cache with cache loader and single
313     * modification per transaction.<br>
314     * This test has never failed up to now.
315     *
316     * @throws Exception Any exception if thrown by the cache.
317     */
318    public void disabledtestReplSyncCacheLoader1Modification() throws Exception
319    {
320       startCache(Configuration.CacheMode.REPL_SYNC, "org.jboss.cache.loader.DummyCacheLoader");
321       performTest(1);
322    }
323
324    /**
325     * Test for a synchronously replicated cache with cache loader and two
326     * modifications per transaction.<br>
327     * This test fails very often with a TimeoutException.
328     *
329     * @throws Exception Any exception if thrown by the cache.
330     */
331    public void testReplSyncCacheLoader2Modifications()
332            throws Exception
333    {
334       startCache(Configuration.CacheMode.REPL_SYNC, "org.jboss.cache.loader.DummyCacheLoader");
335       performTest(2);
336    }
337
338    /**
339     * Perform a single test, using the pre-configured cache.
340     *
341     * @param modificationsPerTx The number of modifications per transaction.
342     * @throws Exception Any exception if thrown by the cache.
343     */
344    private void performTest(int modificationsPerTx) throws Exception
345    {
346       for (int i = 0; i < NUM_RUNS; i++)
347       {
348          if (mcl_exception != null)
349          {
350             // terminate the test on the first failed worker
351 fail("Due to an exception: " + mcl_exception);
352          }
353          // start several worker threads to work with the same set of FQN's
354 Worker[] t = new Worker[NUM_WORKERS];
355          Latch latch = new Latch();
356          for (int j = 0; j < t.length; j++)
357          {
358             t[j] = new Worker(latch, NUM_FQNS_PER_RUN * i,
359                     NUM_FQNS_PER_RUN, modificationsPerTx);
360             t[j].start();
361          }
362          // fire the workers to start processing
363 latch.release();
364          // wait for all workers to complete
365 for (int j = 0; j < t.length; j++)
366          {
367             t[j].join();
368          }
369       }
370    }
371
372    /**
373     * Returns a user transaction to be associated with the calling thread.
374     *
375     * @return A user transaction.
376     * @throws Exception Any exception thrown by the context lookup.
377     */
378    private UserTransaction getTransaction() throws Exception
379    {
380       return (UserTransaction ) new InitialContext (PROPERTIES)
381               .lookup("UserTransaction");
382    }
383
384    /**
385     * Log a message.
386     *
387     * @param msg The meessage to be logged.
388     */
389    private void log(String msg)
390    {
391       System.out.println(System.currentTimeMillis() + " "
392               + Thread.currentThread() + " " + msg);
393    }
394
395    /**
396     * A worker thread that applies the concurrent modifications.
397     *
398     * @author Marian Nikolov
399     * @author $Author: msurtani $
400     * @version $Date: 2006/12/30 19:48:49 $
401     */
402    private class Worker extends Thread
403    {
404       /**
405        * Used to fire all workers at the same time.
406        */
407       private final Latch m_latch;
408       /**
409        * The start id, used as part of the node FQN.
410        */
411       private final int m_start;
412       /**
413        * The number of nodes to create in a single run.
414        */
415       private final int m_count;
416       /**
417        * The number of modifications per single transaction.
418        */
419       private final int m_modificationsPerTx;
420
421       /**
422        * The state of the thread, used for logging.
423        */
424       private int m_state;
425
426       /**
427        * Constructor.
428        *
429        * @param latch Used to synchronize the startup of all worker threads.
430        * @param start The start id.
431        * @param count The number of nodes to create in a single run.
432        * @param modificationsPerTx The number of modifications per
433        * transaction.
434        */
435       public Worker(Latch latch, int start, int count, int modificationsPerTx)
436       {
437          m_latch = latch;
438          m_start = start;
439          m_count = count;
440          m_state = -1;
441          m_modificationsPerTx = modificationsPerTx;
442       }
443
444       /**
445        * {@inheritDoc}
446        */
447       public void run()
448       {
449          try
450          {
451             // the latch shall fire all workers at the same time
452 m_latch.acquire();
453             for (int i = m_start; i < m_start + m_count; i++)
454             {
455                m_state = -1;
456                log("enter " + i);
457                if (checkAndSetState())
458                {
459                   return;
460                }
461                long time = System.currentTimeMillis();
462                UserTransaction tx = getTransaction();
463                tx.begin();
464                if (checkAndSetState())
465                {
466                   return;
467                }
468                // the first worker would create a new node for the FQN
469 // all the others would update the same node
470 Fqn fqn = new Fqn(new Fqn("NODE"), i);
471                for (int m = 0; m < m_modificationsPerTx; m++)
472                {
473                   cache.put(fqn, new Integer (m), new Integer (i));
474                   if (checkAndSetState())
475                   {
476                      return;
477                   }
478                }
479                tx.commit();
480                if (checkAndSetState())
481                {
482                   return;
483                }
484                time = System.currentTimeMillis() - time;
485                log("leave " + i + " took " + time + " msec");
486             }
487          }
488          catch (Exception e)
489          {
490             log("caught exception in state " + m_state + ": " + e);
491             mcl_exception = e;
492          }
493       }
494
495       /**
496        * Checks the current thread and sets it state.
497        *
498        * @return True if the worker has to terminate, false otherwise.
499        */
500       private boolean checkAndSetState()
501       {
502          if (mcl_exception != null)
503          {
504             // another worker failed, terminate
505 log("detected failure in state " + m_state);
506             return true;
507          }
508          m_state++;
509          return false;
510       }
511    }
512
513    public static Test suite()
514    {
515
516       return new TestSuite(ConcurrentCreationDeadlockTest.class);
517
518    }
519
520
521    public static void main(String [] args) throws Exception
522    {
523
524       junit.textui.TestRunner.run(suite());
525
526    }
527 }
528

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags