ReplicatedTransactionDeadlockTest


1   package org.jboss.cache.transaction;
2   
3   import junit.framework.Test;
4   import junit.framework.TestCase;
5   import junit.framework.TestSuite;
6   import org.apache.commons.logging.Log;
7   import org.apache.commons.logging.LogFactory;
8   import org.jboss.cache.CacheImpl;
9   import org.jboss.cache.Fqn;
10  import org.jboss.cache.config.Configuration;
11  import org.jboss.cache.factories.XmlConfigurationParser;
12  
13  import javax.naming.Context  ;
14  import javax.naming.InitialContext  ;
15  import javax.transaction.UserTransaction  ;
16  import java.util.Properties  ;
17  
18  /**
19   * A test created to simulate an unexpected TimeoutException in JBossCache
20   * 1.3.0SP1 (not relevant for 1.2.4 and earlier releases). The error has been
21   * initially observed in a production environment, the test has been created to
22   * simplify the analysis without the complexity of hundreds concurrent
23   * transactions.<br>
24   * This test is relevant for REPL_SYNC mode, (default) isolation level
25   * REPEATABLE_READ and SyncCommitPhase.<br>
26   * The error scenario is:
27   * <ul>
28   * <li>Two caches: SrcCache is the one where we put the modifications, DstCache
29   * only receives the replicated modifications.</li>
30   * <li>Two threads (resp. transactions) A and B modifying the same node X
31   * concurrently on SrcCache.</li>
32   * <li>Let's assume the transaction A is faster in aquiring the lock on X.</li>
33   * <li>Transaction A modifies the node X and starts committing: local prepare,
34   * remote prepare, local commit. The remote prepare will lock the X on DstCache
35   * and it will stay locked until the remote commit releases it later. Note that
36   * in JBossCache 1.3.0SP1 the transaction will release the lock on SrcCache
37   * immediately after the local commit step and before the remote (sync.) commit.
38   * It seems that this have changed between 1.2.4 and 1.3.0 releases.</li>
39   * <li>As soon as the lock on SrcCache is released by A, transaction B aquires
40   * the lock immediately and starts local modifications.<b>Note that in some
41   * cases B is fast enough to do the local prepare and send a remote prepare
42   * message before the remote commit message of A. </b>B is able to do this
43   * because the lock is released by A before its remote commit call.</li>
44   * <li>Now, we have the X locked by A on DstCache waiting for the remote commit
45   * of A to release it. The next messages from SrcCache are in the following
46   * order coming up the JGROUPS stack of the DstCache: remote prepare for B,
47   * remote commit for A.</li>
48   * <li>The remote prepare of B blocks on DstCache, trying to acquire the lock
49   * still held by A.</li>
50   * <li>The remote commit of A waits in the UP queue of the STATE_TRANSFER,
51   * waiting for the previous message (which is the remote prepare of B) to be
52   * processed.</li>
53   * <li>So A cannot be committed because it's blocked by B, which cannot be
54   * prepared, because it's blocked by A, which cannot be committed. Of course the
55   * result is a TimeoutException and too many rolled back transactions.</li>
56   * </ul>
57   *
58   * @author Marian Nikolov
59   * @author $Author: msurtani $
60   * @version $Date: 2006/12/30 17:49:53 $
61   */
62  public class ReplicatedTransactionDeadlockTest extends TestCase
63  {
64  
65     // Number of worker threads
66     private static final int NUM_WORKERS = 2;
67  
68     // The number of test runs to perform.
69     private static final int NUM_RUNS = 100;
70  
71     // useful to increase this to get thread dumps, etc.  Typically should be set to 10,000 though.
72     private static final long LOCK_ACQUISITION_TIMEOUT = 10000;
73  
74     /**
75      * The initial context factory properties.
76      */
77     private static final Properties   PROPERTIES;
78     /**
79      * The context factory to be used for the test.
80      */
81     private static final String   CONTEXT_FACTORY =
82             "org.jboss.cache.transaction.DummyContextFactory";
83     /**
84      * The original context factory to be restored after the test.
85      */
86     private String   m_contextFactory = null;
87  
88     /**
89      * Exception recorded if any of the worker threads fails.
90      */
91     private static volatile Exception   exception = null;
92  
93     /**
94      * The source cache where we put modifications.
95      */
96     private CacheImpl srcCache = null;
97     /**
98      * The target cache where we replicate modifications.
99      */
100    private CacheImpl dstCache = null;
101 
102    private Log log = LogFactory.getLog(ReplicatedTransactionDeadlockTest.class);
103 
104    static
105    {
106       PROPERTIES = new Properties  ();
107       PROPERTIES.put(Context.INITIAL_CONTEXT_FACTORY,
108               "org.jboss.cache.transaction.DummyContextFactory");
109    }
110 
111    /**
112     * Constructor.
113     *
114     * @param name The test name.
115     */
116    public ReplicatedTransactionDeadlockTest(String   name)
117    {
118       super(name);
119    }
120 
121    /**
122     * {@inheritDoc}
123     */
124    protected void setUp() throws Exception  
125    {
126       super.setUp();
127       exception = null;
128       m_contextFactory = System.getProperty(Context.INITIAL_CONTEXT_FACTORY);
129       System.setProperty(Context.INITIAL_CONTEXT_FACTORY, CONTEXT_FACTORY);
130       DummyTransactionManager.getInstance();
131       // setup and start the source cache
132       srcCache = new CacheImpl();
133       srcCache.setConfiguration(new XmlConfigurationParser().parseFile("META-INF/replSync-service.xml"));
134       srcCache.getConfiguration().setTransactionManagerLookupClass(
135               "org.jboss.cache.DummyTransactionManagerLookup");
136       srcCache.getConfiguration().setCacheMode(Configuration.CacheMode.REPL_SYNC);
137 
138       srcCache.getConfiguration().setSyncCommitPhase(true);
139       srcCache.getConfiguration().setLockAcquisitionTimeout(LOCK_ACQUISITION_TIMEOUT);
140       srcCache.create();
141       srcCache.start();
142       // setup and start the destination cache
143       dstCache = new CacheImpl();
144       dstCache.setConfiguration(new XmlConfigurationParser().parseFile("META-INF/replSync-service.xml"));
145       dstCache.getConfiguration().setTransactionManagerLookupClass(
146               "org.jboss.cache.DummyTransactionManagerLookup");
147       dstCache.getConfiguration().setCacheMode(Configuration.CacheMode.REPL_SYNC);
148 
149       dstCache.getConfiguration().setSyncCommitPhase(true);
150       dstCache.getConfiguration().setLockAcquisitionTimeout(LOCK_ACQUISITION_TIMEOUT);
151       dstCache.create();
152       dstCache.start();
153    }
154 
155    /**
156     * {@inheritDoc}
157     */
158    protected void tearDown() throws Exception  
159    {
160       super.tearDown();
161       DummyTransactionManager.destroy();
162       srcCache.stop();
163       srcCache = null;
164       dstCache.stop();
165       dstCache = null;
166       if (m_contextFactory != null)
167       {
168          System.setProperty(Context.INITIAL_CONTEXT_FACTORY,
169                  m_contextFactory);
170          m_contextFactory = null;
171       }
172    }
173 
174    /**
175     * Test for a synchronously replicated cache with concurrent transactions on
176     * the same node.<br>
177     * This test fails very often with a TimeoutException.
178     *
179     * @throws Exception Any exception if thrown by the cache.
180     */
181    public void testConcurrentReplicatedTransaction() throws Exception  
182    {
183       performTest();
184    }
185 
186    /**
187     * Perform a single test, using the pre-configured cache.
188     *
189     * @throws Exception Any exception if thrown by the cache.
190     */
191    private void performTest() throws Exception  
192    {
193       // repeat the test several times since it's not always reproducible
194       for (int i = 0; i < NUM_RUNS; i++)
195       {
196          if (exception != null)
197          {
198             // terminate the test on the first failed worker
199             fail("Due to an exception: " + exception);
200          }
201          // start several worker threads to work with the same FQN
202          Worker[] t = new Worker[NUM_WORKERS];
203          for (int j = 0; j < t.length; j++)
204          {
205             t[j] = new Worker("worker " + i + ":" + j);
206             t[j].start();
207          }
208          // wait for all workers to complete before repeating the test
209          for (Worker aT : t) aT.join();
210       }
211    }
212 
213    /**
214     * Returns a user transaction to be associated with the calling thread.
215     *
216     * @return A user transaction.
217     * @throws Exception Any exception thrown by the context lookup.
218     */
219    private UserTransaction   getTransaction() throws Exception  
220    {
221       return (UserTransaction  ) new InitialContext  (PROPERTIES)
222               .lookup("UserTransaction");
223    }
224 
225    /**
226     * A worker thread that applies the concurrent modifications.
227     *
228     * @author Marian Nikolov
229     * @author $Author: msurtani $
230     * @version $Date: 2006/12/30 17:49:53 $
231     */
232    private class Worker extends Thread  
233    {
234       /**
235        * Constructor.
236        */
237       public Worker(String   name)
238       {
239          super(name);
240       }
241 
242       /**
243        * {@inheritDoc}
244        */
245       public void run()
246       {
247          try
248          {
249             UserTransaction   tx = getTransaction();
250             log.warn("begin");
251             tx.begin();
252             log.warn("put");
253             srcCache.put(new Fqn("Node"), Boolean.FALSE, Boolean.TRUE);
254             log.warn("commit");
255             tx.commit();
256             log.warn("leave");
257          }
258          catch (Exception   e)
259          {
260             log.error("caught exception " + e, e);
261             exception = e;
262          }
263       }
264    }
265 
266    public static Test suite()
267    {
268       return new TestSuite(ReplicatedTransactionDeadlockTest.class);
269    }
270 
271    public static void main(String  [] args) throws Exception  
272    {
273       junit.textui.TestRunner.run(suite());
274    }
275 }
276
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags