Java API By Example, From Geeks To Geeks.

1 /* 2   * JBoss, Home of Professional Open Source 3   * Copyright 2005, JBoss Inc., and individual contributors as indicated 4   * by the @authors tag. See the copyright.txt in the distribution for a 5   * full listing of individual contributors. 6   * 7   * This is free software; you can redistribute it and/or modify it 8   * under the terms of the GNU Lesser General Public License as 9   * published by the Free Software Foundation; either version 2.1 of 10   * the License, or (at your option) any later version. 11   * 12   * This software is distributed in the hope that it will be useful, 13   * but WITHOUT ANY WARRANTY; without even the implied warranty of 14   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15   * Lesser General Public License for more details. 16   * 17   * You should have received a copy of the GNU Lesser General Public 18   * License along with this software; if not, write to the Free 19   * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 20   * 02110-1301 USA, or see the FSF site: http://www.fsf.org. 21   */ 22 package org.jboss.test.recover.test; 23 24 import junit.framework.Test; 25 import junit.framework.TestCase; 26 import junit.framework.TestSuite; 27 28 import org.jboss.logging.Logger; 29 import org.jboss.test.recover.interfaces.DummyRecoverable; 30 import org.jboss.test.recover.interfaces.DummyXAResource; 31 import org.jboss.test.recover.interfaces.DummyXAResourceImpl; 32 import org.jboss.test.recover.interfaces.MockLogger; 33 import org.jboss.tm.TxManager; 34 import org.jboss.tm.TransactionImpl; 35 import org.jboss.tm.XidFactoryBase; 36 import org.jboss.tm.recovery.CorruptedLogRecordException; 37 import org.jboss.tm.recovery.LogRecord; 38 import org.jboss.tm.recovery.RecoveryLogReader; 39 import org.jboss.tm.recovery.RecoveryManager; 40 import org.jboss.tm.recovery.RecoveryTestingException; 41 42 import EDU.oswego.cs.dl.util.concurrent.Semaphore; 43 44 import javax.transaction.SystemException ; 45 import javax.transaction.Transaction ; 46 import javax.transaction.xa.XAException ; 47 48 import java.util.ArrayList ; 49 import java.io.File ; 50 import java.io.IOException ; 51 import java.io.RandomAccessFile ; 52 53 /** 54  * Stand-alone (in-VM) recoverability test. 55  * 56  * @author <a HREF="mailto:bill@jboss.org">Bill Burke</a> 57  * @author <a HREF="mailto:reverbel@ime.usp.br">Francisco Reverbel</a> 58  * @version $Revision: 37406 $ 59  */ 60 public class RecoverabilityTestCase extends TestCase 61 { 62    private Logger traceLog = Logger.getLogger(this.getClass()); 63    private String dirName = "recovery-test"; 64    private String [] directoryList = {dirName}; 65    private int logFileSize = 1000; 66    private String heuristicDir = "heuristic-dir"; 67    private XidFactoryBase xidFactory = TransactionImpl.defaultXidFactory(); 68    private final int N = 5; 69 70    public RecoverabilityTestCase(String s) 71    { 72       super(s); 73        74       xidFactory.setBaseGlobalId("localhost:1099/"); 75       xidFactory.setBranchQualifier("localhost:1099"); 76    } 77     78    public void testFailureAfterTxCommitted() throws Exception 79    { 80       traceLog.info(">>> testFailureAfterTxCommitted()"); 81        82       MockLogger logger = setupTxLogger(); 83       TxManager tm = TxManager.getInstance(); 84        85       DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 86       DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 87       DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 88 89       DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 90       DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 91       DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 92 93       ArrayList recoverables = new ArrayList (); 94       recoverables.add(rec1); 95       recoverables.add(rec2); 96       recoverables.add(rec3); 97 98       logger.setFailAfterCommitting(true); 99       logger.setFailAfter(N); 100        101       try 102       { 103          for (int i = 0; ;i++) 104          { 105             tm.begin(); 106             Transaction tx = tm.getTransaction(); 107             tx.enlistResource(xa1); 108             tx.enlistResource(xa2); 109             tx.enlistResource(xa3); 110             tm.commit(); 111          } 112       } 113       catch (RecoveryTestingException e) 114       { 115          traceLog.info("Expected exception: " + e); 116       } 117       catch (Throwable e) 118       { 119          traceLog.info("Unexpected throwable:", e); 120          fail("Unexpected throwable: " + e); 121       } 122 123       tm.suspend(); // to disassociate thread with any txs. 124 125       logger.stop(); 126 127       int xa1Count = xa1.getCommittedCount(); 128       int xa2Count = xa2.getCommittedCount(); 129       int xa3Count = xa3.getCommittedCount(); 130 131       try 132       { 133          logger = switchTxLogger(); 134          RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 135          manager.recover(recoverables); 136          logger.stop(); 137       } 138       catch (Throwable e) 139       { 140          traceLog.info("Unexpected throwable:", e); 141          fail("Unexpected throwable: " + e); 142       } 143 144       int xa1NextCount = xa1.getCommittedCount(); 145       int xa2NextCount = xa2.getCommittedCount(); 146       int xa3NextCount = xa3.getCommittedCount(); 147 148       assertEquals(N, xa1Count); 149       assertEquals(N, xa2Count); 150       assertEquals(N, xa3Count); 151 152       assertEquals(N + 1, xa1NextCount); 153       assertEquals(N + 1, xa2NextCount); 154       assertEquals(N + 1, xa3NextCount); 155 156       Thread.sleep(1000); // give time to cleanup for logger 157 158       cleanLogDir(dirName); 159 160       traceLog.info("DONE TEST"); 161    } 162 163    public void testFailureBeforeTxCommitted() throws Exception 164    { 165       traceLog.info(">>> testFailureBeforeTxCommitted()"); 166 167       MockLogger logger = setupTxLogger(); 168       TxManager tm = TxManager.getInstance(); 169        170       DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 171       DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 172       DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 173 174       DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 175       DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 176       DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 177 178       ArrayList recoverables = new ArrayList (); 179       recoverables.add(rec1); 180       recoverables.add(rec2); 181       recoverables.add(rec3); 182 183       logger.setFailBeforeCommitting(true); 184       logger.setFailBefore(N); 185     186       try 187       { 188          for (int i = 0; ;i++) 189          { 190             tm.begin(); 191             Transaction tx = tm.getTransaction(); 192             tx.enlistResource(xa1); 193             tx.enlistResource(xa2); 194             tx.enlistResource(xa3); 195             tm.commit(); 196          } 197       } 198       catch (RecoveryTestingException e) 199       { 200          traceLog.info("Expected exception: " + e); 201       } 202       catch (Throwable e) 203       { 204          traceLog.info("Unexpected throwable:", e); 205          fail("Unexpected throwable: " + e); 206       } 207       tm.suspend(); // to disassociate thread with any txs. 208 209       logger.stop(); 210 211       Thread.sleep(1000); // give time to cleanup for logger 212 213       int xa1Count = xa1.getCommittedCount(); 214       int xa2Count = xa2.getCommittedCount(); 215       int xa3Count = xa3.getCommittedCount(); 216 217       int xa1RollbackCount = xa1.getRollbackCount(); 218       int xa2RollbackCount = xa2.getRollbackCount(); 219       int xa3RollbackCount = xa3.getRollbackCount(); 220 221       try 222       { 223          logger = switchTxLogger(); 224          RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 225          manager.recover(recoverables); 226          logger.stop(); 227       } 228       catch (Throwable e) 229       { 230          traceLog.info("Unexpected throwable:", e); 231          fail("Unexpected throwable: " + e); 232       } 233 234       int xa1NextCount = xa1.getCommittedCount(); 235       int xa2NextCount = xa2.getCommittedCount(); 236       int xa3NextCount = xa3.getCommittedCount(); 237 238       int xa1NextRollbackCount = xa1.getRollbackCount(); 239       int xa2NextRollbackCount = xa2.getRollbackCount(); 240       int xa3NextRollbackCount = xa3.getRollbackCount(); 241 242       assertEquals(N, xa1Count); 243       assertEquals(N, xa2Count); 244       assertEquals(N, xa3Count); 245       assertEquals(N, xa1NextCount); 246       assertEquals(N, xa2NextCount); 247       assertEquals(N, xa3NextCount); 248 249       assertEquals(0, xa1RollbackCount); 250       assertEquals(0, xa2RollbackCount); 251       assertEquals(0, xa3RollbackCount); 252       assertEquals(1, xa1NextRollbackCount); 253       assertEquals(1, xa2NextRollbackCount); 254       assertEquals(1, xa3NextRollbackCount); 255 256       Thread.sleep(1000); // give time to cleanup for logger 257 258       cleanLogDir(dirName); 259 260       traceLog.info("DONE TEST"); 261    } 262 263    public void testCommitWithHeuristics() throws Exception 264    { 265       traceLog.info(">>> testCommitWithHeuristics()"); 266        267       MockLogger logger = setupTxLogger(); 268       TxManager tm = TxManager.getInstance(); 269        270       DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 271       DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 272       DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 273 274       DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 275       DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 276       DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 277 278       ArrayList recoverables = new ArrayList (); 279       recoverables.add(rec1); 280       recoverables.add(rec2); 281       recoverables.add(rec3); 282 283       traceLog.info("testing heuristic rollback in the first XA resource"); 284       try 285       { 286          tm.begin(); 287          Transaction tx = tm.getTransaction(); 288          tx.enlistResource(xa1); 289          tx.enlistResource(xa2); 290          tx.enlistResource(xa3); 291          xa1.setCommitErrorCode(XAException.XA_HEURRB); 292          tm.commit(); 293          fail("HeuristicMixedException expected."); 294       } 295       catch (javax.transaction.HeuristicMixedException e) 296       { 297          traceLog.info("Expected exception: " + e); 298          xa1.clearCommitErrorCode(); 299       } 300       catch (Throwable e) 301       { 302          traceLog.info("Unexpected throwable:", e); 303          fail("Unexpected throwable: " + e); 304       } 305 306       traceLog.info("testing heuristic rollback in the second XA resource"); 307       try 308       { 309          tm.begin(); 310          Transaction tx = tm.getTransaction(); 311          tx.enlistResource(xa1); 312          tx.enlistResource(xa2); 313          tx.enlistResource(xa3); 314          xa2.setCommitErrorCode(XAException.XA_HEURRB); 315          tm.commit(); 316          fail("HeuristicMixedException expected."); 317       } 318       catch (javax.transaction.HeuristicMixedException e) 319       { 320          traceLog.info("Expected exception: " + e); 321          xa2.clearCommitErrorCode(); 322       } 323       catch (Throwable e) 324       { 325          traceLog.info("Unexpected throwable:", e); 326          fail("Unexpected throwable: " + e); 327       } 328 329       traceLog.info("testing heuristic rollback in the third XA resource"); 330       try 331       { 332          tm.begin(); 333          Transaction tx = tm.getTransaction(); 334          tx.enlistResource(xa1); 335          tx.enlistResource(xa2); 336          tx.enlistResource(xa3); 337          xa3.setCommitErrorCode(XAException.XA_HEURRB); 338          tm.commit(); 339          fail("HeuristicMixedException expected."); 340       } 341       catch (javax.transaction.HeuristicMixedException e) 342       { 343          traceLog.info("Expected exception: " + e); 344          xa3.clearCommitErrorCode(); 345       } 346       catch (Throwable e) 347       { 348          traceLog.info("Unexpected throwable:", e); 349          fail("Unexpected throwable: " + e); 350       } 351        352       traceLog.info("testing heuristic rollback in all three XA resources"); 353       try 354       { 355          tm.begin(); 356          Transaction tx = tm.getTransaction(); 357          tx.enlistResource(xa1); 358          tx.enlistResource(xa2); 359          tx.enlistResource(xa3); 360          xa1.setCommitErrorCode(XAException.XA_HEURRB); 361          xa2.setCommitErrorCode(XAException.XA_HEURRB); 362          xa3.setCommitErrorCode(XAException.XA_HEURRB); 363          tm.commit(); 364          fail("HeuristicRollbackException expected."); 365       } 366       catch (javax.transaction.HeuristicRollbackException e) 367       { 368          traceLog.info("Expected exception: " + e); 369          xa1.clearCommitErrorCode(); 370          xa2.clearCommitErrorCode(); 371          xa3.clearCommitErrorCode(); 372       } 373       catch (Throwable e) 374       { 375          traceLog.info("Unexpected throwable:", e); 376          fail("Unexpected throwable: " + e); 377       } 378 379       int xa1Count = xa1.getCommittedCount(); 380       int xa2Count = xa2.getCommittedCount(); 381       int xa3Count = xa3.getCommittedCount(); 382       int xa1ForgetCount = xa1.getForgetCount(); 383       int xa2ForgetCount = xa2.getForgetCount(); 384       int xa3ForgetCount = xa3.getForgetCount(); 385 386       assertEquals(2, xa1Count); 387       assertEquals(2, xa2Count); 388       assertEquals(2, xa3Count); 389       assertEquals(2, xa1ForgetCount); 390       assertEquals(2, xa2ForgetCount); 391       assertEquals(2, xa3ForgetCount); 392        393       logger.stop(); 394        395       Thread.sleep(1000); // give time to cleanup for logger 396 397       cleanLogDir(dirName); 398 399       traceLog.info("DONE TEST"); 400    } 401     402    public void testFailureAfterCommitWithHeuristics() throws Exception 403    { 404       traceLog.info(">>> testFailureAfterCommitWithHeuristics"); 405        406       MockLogger logger = setupTxLogger(); 407       TxManager tm = TxManager.getInstance(); 408 409       DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 410       DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 411       DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 412 413       DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 414       DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 415       DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 416 417       ArrayList recoverables = new ArrayList (); 418       recoverables.add(rec1); 419       recoverables.add(rec2); 420       recoverables.add(rec3); 421 422       logger.setFailAfterSavingHeuristicStatus(true); 423 424       try 425       { 426          tm.begin(); 427          Transaction tx = tm.getTransaction(); 428          tx.enlistResource(xa1); 429          tx.enlistResource(xa2); 430          tx.enlistResource(xa3); 431          xa2.setCommitErrorCode(XAException.XA_HEURRB); 432          tm.commit(); 433          fail("RecoveryTestingException expected."); 434       } 435       catch (RecoveryTestingException e) 436       { 437          traceLog.info("Expected exception: " + e); 438          //e.printStackTrace(); 439 } 440       catch (Throwable e) 441       { 442          traceLog.info("Unexpected throwable:", e); 443          fail("Unexpected throwable: " + e); 444       } 445       tm.suspend(); // to disassociate thread with any txs. 446 447       logger.stop(); 448 449       int xa1Count = xa1.getCommittedCount(); 450       int xa2Count = xa2.getCommittedCount(); 451       int xa3Count = xa3.getCommittedCount(); 452       int xa1ForgetCount = xa1.getForgetCount(); 453       int xa2ForgetCount = xa2.getForgetCount(); 454       int xa3ForgetCount = xa3.getForgetCount(); 455 456       assertEquals(1, xa1Count); 457       assertEquals(0, xa2Count); 458       assertEquals(1, xa3Count); 459       assertEquals(0, xa1ForgetCount); 460       assertEquals(0, xa2ForgetCount); 461       assertEquals(0, xa3ForgetCount); 462        463       try 464       { 465          logger = switchTxLogger(); 466          RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 467          manager.recover(recoverables); 468          logger.stop(); 469       } 470       catch (Throwable e) 471       { 472          traceLog.info("Unexpected throwable:", e); 473          fail("Unexpected throwable: " + e); 474       } 475 476       int xa1NextCount = xa1.getCommittedCount(); 477       int xa2NextCount = xa2.getCommittedCount(); 478       int xa3NextCount = xa3.getCommittedCount(); 479       int xa1NextForgetCount = xa1.getForgetCount(); 480       int xa2NextForgetCount = xa2.getForgetCount(); 481       int xa3NextForgetCount = xa3.getForgetCount(); 482 483       assertEquals(1, xa1NextCount); 484       assertEquals(0, xa2NextCount); 485       assertEquals(1, xa3NextCount); 486       assertEquals(0, xa1NextForgetCount); 487       assertEquals(1, xa2NextForgetCount); 488       assertEquals(0, xa3NextForgetCount); 489        490       Thread.sleep(1000); // give time to cleanup for logger 491 492       cleanLogDir(dirName); 493 494       traceLog.info("DONE TEST"); 495    } 496     497    public void testXARetriesInCommit() throws Exception 498    { 499       traceLog.info(">>> testXARetriesInCommit()"); 500 501       traceLog.info("testing commit with XA retries but no TM failure"); 502        503       MockLogger logger = setupTxLogger(); 504       TxManager tm = TxManager.getInstance(); 505        506       DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 507       DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 508       DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 509 510       DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 511       DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 512       DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 513 514       ArrayList recoverables = new ArrayList (); 515       recoverables.add(rec1); 516       recoverables.add(rec2); 517       recoverables.add(rec3); 518 519       try 520       { 521          tm.setXARetryTimeout(1); // one second 522 tm.begin(); 523          Transaction tx = tm.getTransaction(); 524          tx.enlistResource(xa1); 525          tx.enlistResource(xa2); 526          tx.enlistResource(xa3); 527          xa2.setCommitErrorCode(XAException.XA_RETRY); 528          traceLog.info("Expected XAException.XA_RETRY" + 529                        " when committing resources..."); 530          tm.commit(); 531       } 532       catch (Throwable e) 533       { 534          traceLog.info("Unexpected throwable:", e); 535          fail("Unexpected throwable: " + e); 536       } 537        538       int xa1Count = xa1.getCommittedCount(); 539       int xa2Count = xa2.getCommittedCount(); 540       int xa3Count = xa3.getCommittedCount(); 541 542       assertEquals(1, xa1Count); 543       assertEquals(0, xa2Count); 544       assertEquals(1, xa3Count); 545 546       traceLog.info("Expected XAException.XA_RETRY" + 547                     " when retrying commit on resource..."); 548       Thread.sleep(1200); // 1.2 seconds, to let one attempt to commit xa2 fail 549 550       xa2.clearCommitErrorCode(); // next attempt to commit xa2 will succeed 551 552       traceLog.info("XAException.XA_RETRY *NOT* expected..."); 553       Thread.sleep(1200); // 1.2 seconds, to let the next attempt happen 554 555       xa1Count = xa1.getCommittedCount(); 556       xa2Count = xa2.getCommittedCount(); 557       xa3Count = xa3.getCommittedCount(); 558 559       assertEquals(1, xa1Count); 560       assertEquals(1, xa2Count); 561       assertEquals(1, xa3Count); 562        563       logger.stop(); 564        565       Thread.sleep(1000); // give time to cleanup for logger 566 567       cleanLogDir(dirName); 568 569       traceLog.info("DONE TEST"); 570    } 571     572    public void testFailureAfterTxCommittedWithSuccessfulXARetryAtRecoveryTime() 573       throws Exception 574    { 575       traceLog.info(">>> testFailureAfterTxCommittedWithSuccessfulXARetryAtRecoveryTime()"); 576        577       MockLogger logger = setupTxLogger(); 578       TxManager tm = TxManager.getInstance(); 579        580       DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 581       DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 582       DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 583 584       DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 585       DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 586       DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 587 588       ArrayList recoverables = new ArrayList (); 589       recoverables.add(rec1); 590       recoverables.add(rec2); 591       recoverables.add(rec3); 592 593       try 594       { 595          tm.setXARetryTimeout(1); // one second 596 tm.begin(); 597          Transaction tx = tm.getTransaction(); 598          tx.enlistResource(xa1); 599          tx.enlistResource(xa2); 600          tx.enlistResource(xa3); 601          xa2.setCommitErrorCode(XAException.XA_RETRY); 602          traceLog.info("Expected XAException.XA_RETRY" + 603                        " when committing resources..."); 604          tm.commit(); 605       } 606       catch (Throwable e) 607       { 608          traceLog.info("Unexpected throwable:", e); 609          fail("Unexpected throwable: " + e); 610       } 611        612       int xa1Count = xa1.getCommittedCount(); 613       int xa2Count = xa2.getCommittedCount(); 614       int xa3Count = xa3.getCommittedCount(); 615 616       assertEquals(1, xa1Count); 617       assertEquals(0, xa2Count); 618       assertEquals(1, xa3Count); 619        620       tm.suspend(); // to disassociate thread with any txs. 621 622       logger.stop(); 623 624       try 625       { 626          logger = switchTxLogger(); 627          xa2.clearCommitErrorCode(); // next attempt to commit xa2 will succeed 628 RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 629          traceLog.info("XAException.XA_RETRY *NOT* expected during recovery..."); 630          manager.recover(recoverables); 631          logger.stop(); 632       } 633       catch (Throwable e) 634       { 635          traceLog.info("Unexpected throwable:", e); 636          fail("Unexpected throwable: " + e); 637       } 638 639       int xa1NextCount = xa1.getCommittedCount(); 640       int xa2NextCount = xa2.getCommittedCount(); 641       int xa3NextCount = xa3.getCommittedCount(); 642 643       assertEquals(1, xa1NextCount); 644       assertEquals(1, xa2NextCount); 645       assertEquals(1, xa3NextCount); 646        647       Thread.sleep(1000); // give time to cleanup for logger 648 649       cleanLogDir(dirName); 650 651       traceLog.info("DONE TEST"); 652    } 653     654    public void testFailureAfterCommitWithFailedXARetryAtRecoveryTime() 655       throws Exception 656    { 657       traceLog.info(">>> testFailureAfterCommitWithFailedXARetryAtRecoveryTime()"); 658 659       MockLogger logger = setupTxLogger(); 660       TxManager tm = TxManager.getInstance(); 661        662       DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 663       DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 664       DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 665 666       DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 667       DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 668       DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 669 670       ArrayList recoverables = new ArrayList (); 671       recoverables.add(rec1); 672       recoverables.add(rec2); 673       recoverables.add(rec3); 674 675       try 676       { 677          tm.setXARetryTimeout(1); // one second 678 tm.begin(); 679          Transaction tx = tm.getTransaction(); 680          tx.enlistResource(xa1); 681          tx.enlistResource(xa2); 682          tx.enlistResource(xa3); 683          xa2.setCommitErrorCode(XAException.XA_RETRY); 684          traceLog.info("Expected XAException.XA_RETRY" + 685                        " when committing resources..."); 686          tm.commit(); 687       } 688       catch (Throwable e) 689       { 690          traceLog.info("Unexpected throwable:", e); 691          fail("Unexpected throwable: " + e); 692       } 693        694       int xa1Count = xa1.getCommittedCount(); 695       int xa2Count = xa2.getCommittedCount(); 696       int xa3Count = xa3.getCommittedCount(); 697 698       assertEquals(1, xa1Count); 699       assertEquals(0, xa2Count); 700       assertEquals(1, xa3Count); 701        702       tm.suspend(); // to disassociate thread with any txs. 703 704       logger.stop(); 705 706       try 707       { 708          logger = switchTxLogger(); 709          RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 710          traceLog.info("Expected XAException.XA_RETRY during recovery..."); 711          manager.recover(recoverables); 712       } 713       catch (Throwable e) 714       { 715          traceLog.info("Unexpected throwable:", e); 716          fail("Unexpected throwable: " + e); 717       } 718 719       traceLog.info("Expected XAException.XA_RETRY" + 720                     " when retrying commit on resource..."); 721       Thread.sleep(1200); 722 723       xa1Count = xa1.getCommittedCount(); 724       xa2Count = xa2.getCommittedCount(); 725       xa3Count = xa3.getCommittedCount(); 726 727       assertEquals(1, xa1Count); 728       assertEquals(0, xa2Count); 729       assertEquals(1, xa3Count); 730 731       xa2.clearCommitErrorCode(); // next attempt to commit xa2 will succeed 732 traceLog.info("XAException.XA_RETRY *NOT* expected anymore..."); 733       Thread.sleep(1200); 734 735       int xa1NextCount = xa1.getCommittedCount(); 736       int xa2NextCount = xa2.getCommittedCount(); 737       int xa3NextCount = xa3.getCommittedCount(); 738 739       assertEquals(1, xa1NextCount); 740       assertEquals(1, xa2NextCount); 741       assertEquals(1, xa3NextCount); 742 743       logger.stop(); 744        745       Thread.sleep(1000); // give time to cleanup for logger 746 747       cleanLogDir(dirName); 748 749       traceLog.info("DONE TEST"); 750    } 751 752    // The test below assumes that the XAResources are prepared in the 753 // same order that they were enlisted. This is true for our current 754 // TransactionImpl, but will change if/when we move on to an implementation 755 // that prepares XAResources in parallel. 756 public void testFailureAfterOneResourceButNotAllHavePrepared() 757       throws Exception 758    { 759       traceLog.info(">>> testFailureAfterOneResourceButNotAllHavePrepared()"); 760        761       MockLogger logger = setupTxLogger(); 762       TxManager tm = TxManager.getInstance(); 763        764       DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 765       DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 766       DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 767 768       DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 769       DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 770       DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 771 772       ArrayList recoverables = new ArrayList (); 773       recoverables.add(rec1); 774       recoverables.add(rec2); 775       recoverables.add(rec3); 776 777       try 778       { 779          tm.begin(); 780          Transaction tx = tm.getTransaction(); 781          tx.enlistResource(xa1); 782          tx.enlistResource(xa2); 783          tx.enlistResource(xa3); 784          xa2.setPrepareException(new RecoveryTestingException( 785                                  "FAILURE WHEN XA2 WAS ABOUT TO PREPARE")); 786          traceLog.info("Expected RecoveryTestingException..."); 787          tm.commit(); 788          fail("RecoveryTestingException expected."); 789       } 790       catch (RecoveryTestingException e) 791       { 792          traceLog.info("Expected exception: " + e); 793          xa2.clearPrepareException(); 794       } 795       catch (Throwable e) 796       { 797          traceLog.info("Unexpected throwable:", e); 798          fail("Unexpected throwable: " + e); 799       } 800        801       int xa1Count = xa1.getPreparedCount(); 802       int xa2Count = xa2.getPreparedCount(); 803       int xa3Count = xa3.getPreparedCount(); 804 805       assertEquals(1, xa1Count); 806       assertEquals(0, xa2Count); 807       assertEquals(0, xa3Count); 808        809       tm.suspend(); // to disassociate thread with any txs. 810 811       logger.stop(); 812 813       try 814       { 815          logger = switchTxLogger(); 816          RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 817          manager.recover(recoverables); 818          logger.stop(); 819       } 820       catch (Throwable e) 821       { 822          traceLog.info("Unexpected throwable:", e); 823          fail("Unexpected throwable: " + e); 824       } 825 826       int xa1NextCount = xa1.getPreparedCount(); 827       int xa2NextCount = xa2.getPreparedCount(); 828       int xa3NextCount = xa3.getPreparedCount(); 829 830       assertEquals(1, xa1NextCount); 831       assertEquals(0, xa2NextCount); 832       assertEquals(0, xa3NextCount); 833        834       int xa1RollbackCount = xa1.getRollbackCount(); 835       int xa2RollbackCount = xa2.getRollbackCount(); 836       int xa3RollbackCount = xa3.getRollbackCount(); 837 838       assertEquals(1, xa1RollbackCount); 839       assertEquals(0, xa2RollbackCount); 840       assertEquals(0, xa3RollbackCount); 841        842       Thread.sleep(1000); // give time to cleanup for logger 843 844       cleanLogDir(dirName); 845 846       traceLog.info("DONE TEST"); 847    } 848 849    // The test below assumes that the XAResources are committed in the 850 // same order that they were enlisted. This is true for our current 851 // TransactionImpl, but will change if/when we move on to an implementation 852 // that commits XAResources in parallel. 853 public void testFailureAfterOneResourceButNotAllHaveCommitted() 854       throws Exception 855    { 856       traceLog.info(">>> testFailureAfterOneResourceButNotAllHaveCommitted()"); 857        858       MockLogger logger = setupTxLogger(); 859       TxManager tm = TxManager.getInstance(); 860        861       DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 862       DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 863       DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 864 865       DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 866       DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 867       DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 868 869       ArrayList recoverables = new ArrayList (); 870       recoverables.add(rec1); 871       recoverables.add(rec2); 872       recoverables.add(rec3); 873 874       try 875       { 876          tm.begin(); 877          Transaction tx = tm.getTransaction(); 878          tx.enlistResource(xa1); 879          tx.enlistResource(xa2); 880          tx.enlistResource(xa3); 881          xa2.setCommitException(new RecoveryTestingException( 882                                  "FAILURE WHEN XA2 WAS ABOUT TO COMMIT")); 883          traceLog.info("Expected RecoveryTestingException..."); 884          tm.commit(); 885          fail("RecoveryTestingException expected."); 886       } 887       catch (RecoveryTestingException e) 888       { 889          traceLog.info("Expected exception: " + e); 890          xa2.clearCommitException(); 891       } 892       catch (Throwable e) 893       { 894          traceLog.info("Unexpected throwable:", e); 895          fail("Unexpected throwable: " + e); 896       } 897        898       int xa1Count = xa1.getCommittedCount(); 899       int xa2Count = xa2.getCommittedCount(); 900       int xa3Count = xa3.getCommittedCount(); 901 902       assertEquals(1, xa1Count); 903       assertEquals(0, xa2Count); 904       assertEquals(0, xa3Count); 905        906       tm.suspend(); // to disassociate thread with any txs. 907 908       logger.stop(); 909 910       try 911       { 912          logger = switchTxLogger(); 913          RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 914          manager.recover(recoverables); 915          logger.stop(); 916       } 917       catch (Throwable e) 918       { 919          traceLog.info("Unexpected throwable:", e); 920          fail("Unexpected throwable: " + e); 921       } 922 923       int xa1NextCount = xa1.getCommittedCount(); 924       int xa2NextCount = xa2.getCommittedCount(); 925       int xa3NextCount = xa3.getCommittedCount(); 926 927       assertEquals(1, xa1NextCount); 928       assertEquals(1, xa2NextCount); 929       assertEquals(1, xa3NextCount); 930        931       Thread.sleep(1000); // give time to cleanup for logger 932 933       cleanLogDir(dirName); 934 935       traceLog.info("DONE TEST"); 936    } 937 938    // The test below assumes that the XAResources are prepared/committed in the 939 // same order that they were enlisted. This is true for our current 940 // TransactionImpl, but will change if/when we move on to an implementation 941 // that prepares/commits XAResources in parallel. 942 public void testMultipleFailuresInTheSameLogFile() 943       throws Exception 944    { 945       traceLog.info(">>> testMultipleFailuresInTheSameLogFile"); 946        947       MockLogger logger = setupTxLogger(); 948       TxManager tm = TxManager.getInstance(); 949       ArrayList recoverables = new ArrayList (); 950        951       traceLog.info("--- will force a failure after tx committed"); 952        953       DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 954       DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 955       DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 956 957       DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 958       DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 959       DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 960 961       recoverables.add(rec1); 962       recoverables.add(rec2); 963       recoverables.add(rec3); 964 965       logger.setFailAfterCommitting(true); 966       logger.setFailAfter(N); 967        968       try 969       { 970          for (int i = 0; ;i++) 971          { 972             tm.begin(); 973             Transaction tx = tm.getTransaction(); 974             tx.enlistResource(xa1); 975             tx.enlistResource(xa2); 976             tx.enlistResource(xa3); 977             tm.commit(); 978          } 979       } 980       catch (RecoveryTestingException e) 981       { 982          traceLog.info("Expected exception: " + e); 983          logger.setFailAfterCommitting(false); 984       } 985       catch (Throwable e) 986       { 987          traceLog.info("Unexpected throwable:", e); 988          fail("Unexpected throwable: " + e); 989       } 990 991       int xa1Count = xa1.getCommittedCount(); 992       int xa2Count = xa2.getCommittedCount(); 993       int xa3Count = xa3.getCommittedCount(); 994 995       assertEquals(N, xa1Count); 996       assertEquals(N, xa2Count); 997       assertEquals(N, xa3Count); 998 999       tm.suspend(); // to disassociate thread with any txs. 1000 tm.clear(); // clear TM before forcing next failure 1001 1002      traceLog.info("--- will force a failure before tx committed"); 1003       1004      DummyXAResourceImpl xa4 = new DummyXAResourceImpl(); 1005      DummyXAResourceImpl xa5 = new DummyXAResourceImpl(); 1006      DummyXAResourceImpl xa6 = new DummyXAResourceImpl(); 1007 1008      DummyRecoverable rec4 = new DummyRecoverable("four", xa4); 1009      DummyRecoverable rec5 = new DummyRecoverable("five", xa5); 1010      DummyRecoverable rec6 = new DummyRecoverable("six", xa6); 1011 1012      recoverables.add(rec4); 1013      recoverables.add(rec5); 1014      recoverables.add(rec6); 1015 1016      logger.setFailBeforeCommitting(true); 1017      logger.setFailBefore(N); 1018    1019      try 1020      { 1021         for (int i = 0; ;i++) 1022         { 1023            tm.begin(); 1024            Transaction tx = tm.getTransaction(); 1025            tx.enlistResource(xa4); 1026            tx.enlistResource(xa5); 1027            tx.enlistResource(xa6); 1028            tm.commit(); 1029         } 1030      } 1031      catch (RecoveryTestingException e) 1032      { 1033         traceLog.info("Expected exception: " + e); 1034         logger.setFailBeforeCommitting(false); 1035      } 1036      catch (Throwable e) 1037      { 1038         traceLog.info("Unexpected throwable:", e); 1039         fail("Unexpected throwable: " + e); 1040      } 1041 1042      int xa4Count = xa4.getCommittedCount(); 1043      int xa5Count = xa5.getCommittedCount(); 1044      int xa6Count = xa6.getCommittedCount(); 1045       1046      assertEquals(N, xa4Count); 1047      assertEquals(N, xa5Count); 1048      assertEquals(N, xa6Count); 1049 1050      int xa4RollbackCount = xa4.getRollbackCount(); 1051      int xa5RollbackCount = xa5.getRollbackCount(); 1052      int xa6RollbackCount = xa6.getRollbackCount(); 1053       1054      assertEquals(0, xa4RollbackCount); 1055      assertEquals(0, xa5RollbackCount); 1056      assertEquals(0, xa6RollbackCount); 1057 1058      tm.suspend(); // to disassociate thread with any txs. 1059 tm.clear(); // clear TM before forcing next failure 1060 1061      traceLog.info("--- will force a failure after one resource " + 1062                    "but not all have prepared"); 1063       1064      DummyXAResourceImpl xa7 = new DummyXAResourceImpl(); 1065      DummyXAResourceImpl xa8 = new DummyXAResourceImpl(); 1066      DummyXAResourceImpl xa9 = new DummyXAResourceImpl(); 1067 1068      DummyRecoverable rec7 = new DummyRecoverable("seven", xa7); 1069      DummyRecoverable rec8 = new DummyRecoverable("eigth", xa8); 1070      DummyRecoverable rec9 = new DummyRecoverable("nine", xa9); 1071 1072      recoverables.add(rec7); 1073      recoverables.add(rec8); 1074      recoverables.add(rec9); 1075 1076      try 1077      { 1078         tm.begin(); 1079         Transaction tx = tm.getTransaction(); 1080         tx.enlistResource(xa7); 1081         tx.enlistResource(xa8); 1082         tx.enlistResource(xa9); 1083         xa8.setPrepareException(new RecoveryTestingException( 1084                                 "FAILURE WHEN XA8 WAS ABOUT TO PREPARE")); 1085         traceLog.info("Expected RecoveryTestingException..."); 1086         tm.commit(); 1087         fail("RecoveryTestingException expected."); 1088      } 1089      catch (RecoveryTestingException e) 1090      { 1091         traceLog.info("Expected exception: " + e); 1092         xa8.clearPrepareException(); 1093      } 1094      catch (Throwable e) 1095      { 1096         traceLog.info("Unexpected throwable:", e); 1097         fail("Unexpected throwable: " + e); 1098      } 1099       1100      int xa7PreparedCount = xa7.getPreparedCount(); 1101      int xa8PreparedCount = xa8.getPreparedCount(); 1102      int xa9PreparedCount = xa9.getPreparedCount(); 1103 1104      assertEquals(1, xa7PreparedCount); 1105      assertEquals(0, xa8PreparedCount); 1106      assertEquals(0, xa9PreparedCount); 1107       1108      tm.suspend(); // to disassociate thread with any txs. 1109 tm.clear(); // clear TM before forcing next failure 1110 1111      traceLog.info("--- will force a failure after one resource " + 1112                    "but not all have committed"); 1113       1114      DummyXAResourceImpl xa10 = new DummyXAResourceImpl(); 1115      DummyXAResourceImpl xa11 = new DummyXAResourceImpl(); 1116      DummyXAResourceImpl xa12 = new DummyXAResourceImpl(); 1117 1118      DummyRecoverable rec10 = new DummyRecoverable("ten", xa10); 1119      DummyRecoverable rec11 = new DummyRecoverable("eleven", xa11); 1120      DummyRecoverable rec12 = new DummyRecoverable("twelve", xa12); 1121 1122      recoverables.add(rec10); 1123      recoverables.add(rec11); 1124      recoverables.add(rec12); 1125 1126      try 1127      { 1128         tm.begin(); 1129         Transaction tx = tm.getTransaction(); 1130         tx.enlistResource(xa10); 1131         tx.enlistResource(xa11); 1132         tx.enlistResource(xa12); 1133         xa11.setCommitException(new RecoveryTestingException( 1134                                 "FAILURE WHEN XA11 WAS ABOUT TO COMMIT")); 1135         traceLog.info("Expected RecoveryTestingException..."); 1136         tm.commit(); 1137         fail("RecoveryTestingException expected."); 1138      } 1139      catch (RecoveryTestingException e) 1140      { 1141         traceLog.info("Expected exception: " + e); 1142         xa11.clearCommitException(); 1143      } 1144      catch (Throwable e) 1145      { 1146         traceLog.info("Unexpected throwable:", e); 1147         fail("Unexpected throwable: " + e); 1148      } 1149       1150      int xa10Count = xa10.getCommittedCount(); 1151      int xa11Count = xa11.getCommittedCount(); 1152      int xa12Count = xa12.getCommittedCount(); 1153 1154      assertEquals(1, xa10Count); 1155      assertEquals(0, xa11Count); 1156      assertEquals(0, xa12Count); 1157       1158      tm.suspend(); // to disassociate thread with any txs. 1159 1160      traceLog.info("--- will test recovery with multiple failures " + 1161                    "in the same log file"); 1162 1163      logger.stop(); 1164 1165      try 1166      { 1167         logger = switchTxLogger(); 1168         RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 1169         manager.recover(recoverables); 1170         logger.stop(); 1171      } 1172      catch (Throwable e) 1173      { 1174         traceLog.info("Unexpected throwable:", e); 1175         fail("Unexpected throwable: " + e); 1176      } 1177 1178      int xa1NextCount = xa1.getCommittedCount(); 1179      int xa2NextCount = xa2.getCommittedCount(); 1180      int xa3NextCount = xa3.getCommittedCount(); 1181 1182      assertEquals(N + 1, xa1NextCount); 1183      assertEquals(N + 1, xa2NextCount); 1184      assertEquals(N + 1, xa3NextCount); 1185       1186      int xa4NextCount = xa4.getCommittedCount(); 1187      int xa5NextCount = xa5.getCommittedCount(); 1188      int xa6NextCount = xa6.getCommittedCount(); 1189 1190      assertEquals(N, xa4NextCount); 1191      assertEquals(N, xa5NextCount); 1192      assertEquals(N, xa6NextCount); 1193 1194      int xa4NextRollbackCount = xa4.getRollbackCount(); 1195      int xa5NextRollbackCount = xa5.getRollbackCount(); 1196      int xa6NextRollbackCount = xa6.getRollbackCount(); 1197 1198      assertEquals(1, xa4NextRollbackCount); 1199      assertEquals(1, xa5NextRollbackCount); 1200      assertEquals(1, xa6NextRollbackCount); 1201 1202      int xa7NextPreparedCount = xa7.getPreparedCount(); 1203      int xa8NextPreparedCount = xa8.getPreparedCount(); 1204      int xa9NextPreparedCount = xa9.getPreparedCount(); 1205 1206      assertEquals(1, xa7NextPreparedCount); 1207      assertEquals(0, xa8NextPreparedCount); 1208      assertEquals(0, xa9NextPreparedCount); 1209       1210      int xa7RollbackCount = xa7.getRollbackCount(); 1211      int xa8RollbackCount = xa8.getRollbackCount(); 1212      int xa9RollbackCount = xa9.getRollbackCount(); 1213 1214      assertEquals(1, xa7RollbackCount); 1215      assertEquals(0, xa8RollbackCount); 1216      assertEquals(0, xa9RollbackCount); 1217       1218      int xa10NextCount = xa10.getCommittedCount(); 1219      int xa11NextCount = xa11.getCommittedCount(); 1220      int xa12NextCount = xa12.getCommittedCount(); 1221 1222      assertEquals(1, xa10NextCount); 1223      assertEquals(1, xa11NextCount); 1224      assertEquals(1, xa12NextCount); 1225       1226      Thread.sleep(1000); // give time to cleanup for logger 1227 1228      cleanLogDir(dirName); 1229 1230      traceLog.info("DONE TEST"); 1231   } 1232    1233   // The test below assumes that the XAResources are prepared/committed in the 1234 // same order that they were enlisted. This is true for our current 1235 // TransactionImpl, but will change if/when we move on to an implementation 1236 // that prepares/commits XAResources in parallel. 1237 public void testMultipleConcurrentFailuresInTheSameLogFile() 1238      throws Exception 1239   { 1240      traceLog.info(">>> testMultipleConcurrentFailuresInTheSameLogFile"); 1241       1242      MockLogger logger = setupTxLogger(); 1243      final TxManager tm = TxManager.getInstance(); 1244      ArrayList recoverables = new ArrayList (); 1245      final int nResources = 12; 1246       1247      final DummyXAResourceImpl[] xa = new DummyXAResourceImpl[nResources]; 1248      DummyRecoverable rec[] = new DummyRecoverable[nResources]; 1249      for (int i = 0; i < nResources; i++) 1250      { 1251         xa[i] = new DummyXAResourceImpl(); 1252         rec[i] = new DummyRecoverable("rec" + i, xa[i]); 1253         recoverables.add(rec[i]); 1254      } 1255       1256      final Semaphore s = new Semaphore(0); 1257 1258      Runnable r1 = new Runnable () 1259      { 1260         public void run() 1261         { 1262            traceLog.info("--- started thread that will force a failure " + 1263                          "after tx committed"); 1264            try 1265            { 1266               for (int i = 0; ;i++) 1267               { 1268                  tm.begin(); 1269                  Transaction tx = tm.getTransaction(); 1270                  tx.enlistResource(xa[0]); 1271                  tx.enlistResource(xa[1]); 1272                  tx.enlistResource(xa[2]); 1273                  if (i == N) 1274                     xa[0].setCommitException(new RecoveryTestingException( 1275                        "FAILURE BEFORE COMMITTING xa[0] -- THE COMMIT DECISION" + 1276                        " WAS LOGGED, BUT NO RESOURCES WERE COMMITTED YET")); 1277                  tm.commit(); 1278               } 1279            } 1280            catch (RecoveryTestingException e) 1281            { 1282               traceLog.info("Expected exception: " + e); 1283               xa[0].clearCommitException(); 1284            } 1285            catch (Throwable e) 1286            { 1287               traceLog.info("Unexpected throwable:", e); 1288               fail("Unexpected throwable: " + e); 1289            } 1290 1291            int xa0Count = xa[0].getCommittedCount(); 1292            int xa1Count = xa[1].getCommittedCount(); 1293            int xa2Count = xa[2].getCommittedCount(); 1294 1295            assertEquals(N, xa0Count); 1296            assertEquals(N, xa1Count); 1297            assertEquals(N, xa2Count); 1298 1299            try 1300            { 1301               tm.suspend(); // to disassociate thread with any txs. 1302 } 1303            catch (SystemException e) 1304            { 1305               traceLog.info("Unexpected exception:", e); 1306            } 1307             1308            traceLog.info("--- finished thread that forced a failure " + 1309                          "after tx committed"); 1310             1311            s.release(); 1312         } 1313      }; 1314       1315      Runnable r2 = new Runnable () 1316      { 1317         public void run() 1318         { 1319            traceLog.info("--- started thread that will force a failure " + 1320                          "before tx committed"); 1321             1322            try 1323            { 1324               for (int i = 0; ;i++) 1325               { 1326                  tm.begin(); 1327                  Transaction tx = tm.getTransaction(); 1328                  tx.enlistResource(xa[3]); 1329                  tx.enlistResource(xa[4]); 1330                  tx.enlistResource(xa[5]); 1331                  if (i == N) 1332                     xa[5].setAfterPrepareException(new RecoveryTestingException( 1333                        "FAILURE AFTER PREPARING xa5 -- ALL RESOURCES WERE " + 1334                        "PREPARED BUT NO COMMIT DECISION WAS LOGGED")); 1335                  tm.commit(); 1336               } 1337            } 1338            catch (RecoveryTestingException e) 1339            { 1340               traceLog.info("Expected exception: " + e); 1341               xa[5].clearAfterPrepareException(); 1342            } 1343            catch (Throwable e) 1344            { 1345               traceLog.info("Unexpected throwable:", e); 1346               fail("Unexpected throwable: " + e); 1347            } 1348 1349            int xa3Count = xa[3].getCommittedCount(); 1350            int xa4Count = xa[4].getCommittedCount(); 1351            int xa5Count = xa[5].getCommittedCount(); 1352             1353            assertEquals(N, xa3Count); 1354            assertEquals(N, xa4Count); 1355            assertEquals(N, xa5Count); 1356 1357            int xa3RollbackCount = xa[3].getRollbackCount(); 1358            int xa4RollbackCount = xa[4].getRollbackCount(); 1359            int xa5RollbackCount = xa[5].getRollbackCount(); 1360             1361            assertEquals(0, xa3RollbackCount); 1362            assertEquals(0, xa4RollbackCount); 1363            assertEquals(0, xa5RollbackCount); 1364 1365            try 1366            { 1367               tm.suspend(); // to disassociate thread with any txs. 1368 } 1369            catch (SystemException e) 1370            { 1371               traceLog.info("Unexpected exception:", e); 1372            } 1373             1374            traceLog.info("--- finished thread that forced a failure " + 1375                          "before tx committed"); 1376 1377            s.release(); 1378         } 1379      }; 1380       1381      Runnable r3 = new Runnable () 1382      { 1383         public void run() 1384         { 1385            traceLog.info("--- started thread that will force a failure after" + 1386                          " one resource but not all have prepared"); 1387            try 1388            { 1389               tm.begin(); 1390               Transaction tx = tm.getTransaction(); 1391               tx.enlistResource(xa[6]); 1392               tx.enlistResource(xa[7]); 1393               tx.enlistResource(xa[8]); 1394               xa[7].setPrepareException(new RecoveryTestingException( 1395                                    "FAILURE WHEN xa[7] WAS ABOUT TO PREPARE")); 1396               tm.commit(); 1397               fail("RecoveryTestingException expected."); 1398            } 1399            catch (RecoveryTestingException e) 1400            { 1401               traceLog.info("Expected exception: " + e); 1402               xa[7].clearPrepareException(); 1403            } 1404            catch (Throwable e) 1405            { 1406               traceLog.info("Unexpected throwable:", e); 1407               fail("Unexpected throwable: " + e); 1408            } 1409             1410            int xa6PreparedCount = xa[6].getPreparedCount(); 1411            int xa7PreparedCount = xa[7].getPreparedCount(); 1412            int xa8PreparedCount = xa[8].getPreparedCount(); 1413 1414            assertEquals(1, xa6PreparedCount); 1415            assertEquals(0, xa7PreparedCount); 1416            assertEquals(0, xa8PreparedCount); 1417             1418            try 1419            { 1420               tm.suspend(); // to disassociate thread with any txs. 1421 } 1422            catch (SystemException e) 1423            { 1424               traceLog.info("Unexpected exception:", e); 1425            } 1426             1427            traceLog.info("--- finished thread that forced a failure after" + 1428                          " one resource but not all have prepared"); 1429            s.release(); 1430         } 1431      }; 1432 1433      Runnable r4 = new Runnable () 1434      { 1435         public void run() 1436         { 1437            traceLog.info("--- started thread that will force a failure after" + 1438                          " one resource but not all have committed"); 1439            try 1440            { 1441               tm.begin(); 1442               Transaction tx = tm.getTransaction(); 1443               tx.enlistResource(xa[9]); 1444               tx.enlistResource(xa[10]); 1445               tx.enlistResource(xa[11]); 1446               xa[10].setCommitException(new RecoveryTestingException( 1447                                      "FAILURE WHEN xa10 WAS ABOUT TO COMMIT")); 1448               tm.commit(); 1449               fail("RecoveryTestingException expected."); 1450            } 1451            catch (RecoveryTestingException e) 1452            { 1453               traceLog.info("Expected exception: " + e); 1454               xa[10].clearCommitException(); 1455            } 1456            catch (Throwable e) 1457            { 1458               traceLog.info("Unexpected throwable:", e); 1459               fail("Unexpected throwable: " + e); 1460            } 1461             1462            int xa9Count = xa[9].getCommittedCount(); 1463            int xa10Count = xa[10].getCommittedCount(); 1464            int xa11Count = xa[11].getCommittedCount(); 1465 1466            assertEquals(1, xa9Count); 1467            assertEquals(0, xa10Count); 1468            assertEquals(0, xa11Count); 1469             1470            try 1471            { 1472               tm.suspend(); // to disassociate thread with any txs. 1473 } 1474            catch (SystemException e) 1475            { 1476               traceLog.info("Unexpected exception:", e); 1477            } 1478             1479            traceLog.info("--- finished thread that forced failure after " + 1480                          "one resource but not all have committed"); 1481            s.release(); 1482         } 1483      }; 1484 1485      // start threads that will force failures 1486 new Thread (r1).start(); 1487      new Thread (r2).start(); 1488      new Thread (r3).start(); 1489      new Thread (r4).start(); 1490       1491      // wait untill all threads have finished 1492 s.acquire(); 1493      s.acquire(); 1494      s.acquire(); 1495      s.acquire(); 1496 1497      traceLog.info("--- will test recovery with multiple failures in the same log file"); 1498 1499      logger.stop(); 1500 1501      try 1502      { 1503         logger = switchTxLogger(); 1504         RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 1505         manager.recover(recoverables); 1506         logger.stop(); 1507      } 1508      catch (Throwable e) 1509      { 1510         traceLog.info("Unexpected throwable:", e); 1511         fail("Unexpected throwable: " + e); 1512      } 1513 1514      int xa0NextCount = xa[0].getCommittedCount(); 1515      int xa1NextCount = xa[1].getCommittedCount(); 1516      int xa2NextCount = xa[2].getCommittedCount(); 1517 1518      assertEquals(N + 1, xa0NextCount); 1519      assertEquals(N + 1, xa1NextCount); 1520      assertEquals(N + 1, xa2NextCount); 1521       1522      int xa3NextCount = xa[3].getCommittedCount(); 1523      int xa4NextCount = xa[4].getCommittedCount(); 1524      int xa5NextCount = xa[5].getCommittedCount(); 1525 1526      assertEquals(N, xa3NextCount); 1527      assertEquals(N, xa4NextCount); 1528      assertEquals(N, xa5NextCount); 1529 1530      int xa3NextRollbackCount = xa[3].getRollbackCount(); 1531      int xa4NextRollbackCount = xa[4].getRollbackCount(); 1532      int xa5NextRollbackCount = xa[5].getRollbackCount(); 1533 1534      assertEquals(1, xa3NextRollbackCount); 1535      assertEquals(1, xa4NextRollbackCount); 1536      assertEquals(1, xa5NextRollbackCount); 1537 1538      int xa6NextPreparedCount = xa[6].getPreparedCount(); 1539      int xa7NextPreparedCount = xa[7].getPreparedCount(); 1540      int xa8NextPreparedCount = xa[8].getPreparedCount(); 1541 1542      assertEquals(1, xa6NextPreparedCount); 1543      assertEquals(0, xa7NextPreparedCount); 1544      assertEquals(0, xa8NextPreparedCount); 1545       1546      int xa6RollbackCount = xa[6].getRollbackCount(); 1547      int xa7RollbackCount = xa[7].getRollbackCount(); 1548      int xa8RollbackCount = xa[8].getRollbackCount(); 1549 1550      assertEquals(1, xa6RollbackCount); 1551      assertEquals(0, xa7RollbackCount); 1552      assertEquals(0, xa8RollbackCount); 1553       1554      int xa9NextCount = xa[9].getCommittedCount(); 1555      int xa10NextCount = xa[10].getCommittedCount(); 1556      int xa11NextCount = xa[11].getCommittedCount(); 1557 1558      assertEquals(1, xa9NextCount); 1559      assertEquals(1, xa10NextCount); 1560      assertEquals(1, xa11NextCount); 1561       1562      Thread.sleep(1000); // give time to cleanup for logger 1563 1564      cleanLogDir(dirName); 1565 1566      traceLog.info("DONE TEST"); 1567   } 1568    1569   public void testLogCorruptionInTheChecksumOfTheLastRecord() 1570      throws Exception 1571   { 1572      traceLog.info(">>> testLogCorruptionInTheChecksumOfTheLastRecord()"); 1573       1574      MockLogger logger = setupTxLogger(); 1575      TxManager tm = TxManager.getInstance(); 1576       1577      DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 1578      DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 1579      DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 1580 1581      DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 1582      DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 1583      DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 1584 1585      ArrayList recoverables = new ArrayList (); 1586      recoverables.add(rec1); 1587      recoverables.add(rec2); 1588      recoverables.add(rec3); 1589 1590      logger.setFailAfterCommitting(true); 1591      logger.setFailAfter(N); 1592       1593      try 1594      { 1595         for (int i = 0; ;i++) 1596         { 1597            tm.begin(); 1598            Transaction tx = tm.getTransaction(); 1599            tx.enlistResource(xa1); 1600            tx.enlistResource(xa2); 1601            tx.enlistResource(xa3); 1602            tm.commit(); 1603         } 1604      } 1605      catch (RecoveryTestingException e) 1606      { 1607         traceLog.info("Expected exception: " + e); 1608      } 1609      catch (Throwable e) 1610      { 1611         traceLog.info("Unexpected throwable:", e); 1612         fail("Unexpected throwable: " + e); 1613      } 1614 1615      tm.suspend(); // to disassociate thread with any txs. 1616 1617      logger.stop(); 1618 1619      int xa1Count = xa1.getCommittedCount(); 1620      int xa2Count = xa2.getCommittedCount(); 1621      int xa3Count = xa3.getCommittedCount(); 1622 1623      int xa1RollbackCount = xa1.getRollbackCount(); 1624      int xa2RollbackCount = xa2.getRollbackCount(); 1625      int xa3RollbackCount = xa3.getRollbackCount(); 1626 1627      try 1628      { 1629         logger = switchTxLogger(); 1630          1631         RecoveryLogReader[] readers = logger.getRecoveryLogs(); 1632         for (int i = 0; i < readers.length; i++) 1633         { 1634            File f = new File (readers[i].getLogFileName()); 1635            traceLog.info("log file: " + f.getName() + ", len: " + f.length()); 1636            if (f.length() == logFileSize) 1637            { 1638               RandomAccessFile raf = new RandomAccessFile (f, "rw"); 1639               // Scan backwards and overwrite with a zero 1640 // the last non-zero byte in the log file. 1641 for (long pos = logFileSize - 1; pos >= 0; pos--) 1642               { 1643                  raf.seek(pos); 1644                  byte b = raf.readByte(); 1645                  if (b != 0) 1646                  { 1647                     traceLog.info("Overwriting with a zero the last " + 1648                                   "non-zero byte in log file " + f.getName()); 1649                     raf.seek(pos); 1650                     raf.writeByte(0); 1651                     break; 1652                  } 1653               } 1654            } 1655         } 1656          1657         RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 1658         manager.recover(recoverables); 1659         logger.stop(); 1660      } 1661      catch (CorruptedLogRecordException e) 1662      { 1663         traceLog.info("Expected exception:", e); 1664      } 1665      catch (Throwable e) 1666      { 1667         traceLog.info("Unexpected throwable:", e); 1668         fail("Unexpected throwable: " + e); 1669      } 1670 1671      int xa1NextCount = xa1.getCommittedCount(); 1672      int xa2NextCount = xa2.getCommittedCount(); 1673      int xa3NextCount = xa3.getCommittedCount(); 1674 1675      int xa1NextRollbackCount = xa1.getRollbackCount(); 1676      int xa2NextRollbackCount = xa2.getRollbackCount(); 1677      int xa3NextRollbackCount = xa3.getRollbackCount(); 1678 1679      assertEquals(N, xa1Count); 1680      assertEquals(N, xa2Count); 1681      assertEquals(N, xa3Count); 1682      assertEquals(N, xa1NextCount); 1683      assertEquals(N, xa2NextCount); 1684      assertEquals(N, xa3NextCount); 1685 1686      assertEquals(0, xa1RollbackCount); 1687      assertEquals(0, xa2RollbackCount); 1688      assertEquals(0, xa3RollbackCount); 1689      assertEquals(1, xa1NextRollbackCount); 1690      assertEquals(1, xa2NextRollbackCount); 1691      assertEquals(1, xa3NextRollbackCount); 1692 1693      Thread.sleep(1000); // give time to cleanup for logger 1694 1695      cleanLogDir(dirName); 1696 1697      traceLog.info("DONE TEST"); 1698   } 1699 1700   public void testLogCorruptionInTheHeaderOfTheLastRecord() 1701      throws Exception 1702   { 1703      traceLog.info(">>> testLogCorruptionInTheHeaderOfTheLastRecord()"); 1704       1705      MockLogger logger = setupTxLogger(); 1706      TxManager tm = TxManager.getInstance(); 1707       1708      DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 1709      DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 1710      DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 1711 1712      DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 1713      DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 1714      DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 1715 1716      ArrayList recoverables = new ArrayList (); 1717      recoverables.add(rec1); 1718      recoverables.add(rec2); 1719      recoverables.add(rec3); 1720 1721      logger.setFailAfterCommitting(true); 1722      logger.setFailAfter(N); 1723       1724      try 1725      { 1726         for (int i = 0; ;i++) 1727         { 1728            tm.begin(); 1729            Transaction tx = tm.getTransaction(); 1730            tx.enlistResource(xa1); 1731            tx.enlistResource(xa2); 1732            tx.enlistResource(xa3); 1733            tm.commit(); 1734         } 1735      } 1736      catch (RecoveryTestingException e) 1737      { 1738         traceLog.info("Expected exception: " + e); 1739      } 1740      catch (Throwable e) 1741      { 1742         traceLog.info("Unexpected throwable:", e); 1743         fail("Unexpected throwable: " + e); 1744      } 1745 1746      tm.suspend(); // to disassociate thread with any txs. 1747 1748      logger.stop(); 1749 1750      int xa1Count = xa1.getCommittedCount(); 1751      int xa2Count = xa2.getCommittedCount(); 1752      int xa3Count = xa3.getCommittedCount(); 1753 1754      int xa1RollbackCount = xa1.getRollbackCount(); 1755      int xa2RollbackCount = xa2.getRollbackCount(); 1756      int xa3RollbackCount = xa3.getRollbackCount(); 1757 1758      try 1759      { 1760         logger = switchTxLogger(); 1761          1762         RecoveryLogReader[] readers = logger.getRecoveryLogs(); 1763         for (int i = 0; i < readers.length; i++) 1764         { 1765            File f = new File (readers[i].getLogFileName()); 1766            traceLog.info("log file: " + f.getName() + ", len: " + f.length()); 1767            if (f.length() == logFileSize) 1768            { 1769               RandomAccessFile raf = new RandomAccessFile (f, "rw"); 1770               // Scan backwards to the header of the last log record. 1771 long pos = logFileSize - LogRecord.HEADER_LEN; 1772               while (!match(raf, pos, LogRecord.HEADER)) 1773                    pos--; 1774               traceLog.info("Overwriting with a zero the first header " + 1775                             "byte of the last record in log file " + 1776                             f.getName()); 1777               raf.seek(pos); 1778               raf.writeByte(0); 1779            } 1780         } 1781          1782         RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 1783         manager.recover(recoverables); 1784         logger.stop(); 1785      } 1786      catch (CorruptedLogRecordException e) 1787      { 1788         traceLog.info("Expected exception:", e); 1789      } 1790      catch (Throwable e) 1791      { 1792         traceLog.info("Unexpected throwable:", e); 1793         fail("Unexpected throwable: " + e); 1794      } 1795 1796      int xa1NextCount = xa1.getCommittedCount(); 1797      int xa2NextCount = xa2.getCommittedCount(); 1798      int xa3NextCount = xa3.getCommittedCount(); 1799 1800      int xa1NextRollbackCount = xa1.getRollbackCount(); 1801      int xa2NextRollbackCount = xa2.getRollbackCount(); 1802      int xa3NextRollbackCount = xa3.getRollbackCount(); 1803 1804      assertEquals(N, xa1Count); 1805      assertEquals(N, xa2Count); 1806      assertEquals(N, xa3Count); 1807      assertEquals(N, xa1NextCount); 1808      assertEquals(N, xa2NextCount); 1809      assertEquals(N, xa3NextCount); 1810 1811      assertEquals(0, xa1RollbackCount); 1812      assertEquals(0, xa2RollbackCount); 1813      assertEquals(0, xa3RollbackCount); 1814      assertEquals(1, xa1NextRollbackCount); 1815      assertEquals(1, xa2NextRollbackCount); 1816      assertEquals(1, xa3NextRollbackCount); 1817 1818      Thread.sleep(1000); // give time to cleanup for logger 1819 1820      cleanLogDir(dirName); 1821 1822      traceLog.info("DONE TEST"); 1823   } 1824 1825   public void testLogCorruptionInTheRecLenFieldOfTheLastRecord() 1826      throws Exception 1827   { 1828         traceLog.info(">>> testLogCorruptionInTheRecLenFieldOfTheLastRecord()"); 1829          1830         MockLogger logger = setupTxLogger(); 1831         TxManager tm = TxManager.getInstance(); 1832          1833         DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 1834         DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 1835         DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 1836 1837         DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 1838         DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 1839         DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 1840 1841         ArrayList recoverables = new ArrayList (); 1842         recoverables.add(rec1); 1843         recoverables.add(rec2); 1844         recoverables.add(rec3); 1845 1846         logger.setFailAfterCommitting(true); 1847         logger.setFailAfter(N); 1848          1849         try 1850         { 1851            for (int i = 0; ;i++) 1852            { 1853               tm.begin(); 1854               Transaction tx = tm.getTransaction(); 1855               tx.enlistResource(xa1); 1856               tx.enlistResource(xa2); 1857               tx.enlistResource(xa3); 1858               tm.commit(); 1859            } 1860         } 1861         catch (RecoveryTestingException e) 1862         { 1863            traceLog.info("Expected exception: " + e); 1864         } 1865         catch (Throwable e) 1866         { 1867            traceLog.info("Unexpected throwable:", e); 1868            fail("Unexpected throwable: " + e); 1869         } 1870 1871         tm.suspend(); // to disassociate thread with any txs. 1872 1873         logger.stop(); 1874 1875         int xa1Count = xa1.getCommittedCount(); 1876         int xa2Count = xa2.getCommittedCount(); 1877         int xa3Count = xa3.getCommittedCount(); 1878 1879         int xa1RollbackCount = xa1.getRollbackCount(); 1880         int xa2RollbackCount = xa2.getRollbackCount(); 1881         int xa3RollbackCount = xa3.getRollbackCount(); 1882 1883         try 1884         { 1885            logger = switchTxLogger(); 1886             1887            RecoveryLogReader[] readers = logger.getRecoveryLogs(); 1888            for (int i = 0; i < readers.length; i++) 1889            { 1890               File f = new File (readers[i].getLogFileName()); 1891               traceLog.info("log file: " + f.getName() + ", len: " + f.length()); 1892               if (f.length() == logFileSize) 1893               { 1894                  RandomAccessFile raf = new RandomAccessFile (f, "rw"); 1895                  // Scan backwards to the header of the last log record. 1896 long pos = logFileSize - LogRecord.HEADER_LEN; 1897                  while (!match(raf, pos, LogRecord.HEADER)) 1898                       pos--; 1899                  pos += LogRecord.HEADER_LEN; 1900                  traceLog.info("Changing the first recLen byte of the " + 1901                                "last record in log file " + f.getName()); 1902                  raf.seek(pos); 1903                  byte b = raf.readByte(); 1904                  raf.seek(pos); 1905                  raf.writeByte(b + 1); 1906               } 1907            } 1908             1909            RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 1910            manager.recover(recoverables); 1911            logger.stop(); 1912         } 1913         catch (CorruptedLogRecordException e) 1914         { 1915            traceLog.info("Expected exception:", e); 1916         } 1917         catch (Throwable e) 1918         { 1919            traceLog.info("Unexpected throwable:", e); 1920            fail("Unexpected throwable: " + e); 1921         } 1922 1923         int xa1NextCount = xa1.getCommittedCount(); 1924         int xa2NextCount = xa2.getCommittedCount(); 1925         int xa3NextCount = xa3.getCommittedCount(); 1926 1927         int xa1NextRollbackCount = xa1.getRollbackCount(); 1928         int xa2NextRollbackCount = xa2.getRollbackCount(); 1929         int xa3NextRollbackCount = xa3.getRollbackCount(); 1930 1931         assertEquals(N, xa1Count); 1932         assertEquals(N, xa2Count); 1933         assertEquals(N, xa3Count); 1934         assertEquals(N, xa1NextCount); 1935         assertEquals(N, xa2NextCount); 1936         assertEquals(N, xa3NextCount); 1937 1938         assertEquals(0, xa1RollbackCount); 1939         assertEquals(0, xa2RollbackCount); 1940         assertEquals(0, xa3RollbackCount); 1941         assertEquals(1, xa1NextRollbackCount); 1942         assertEquals(1, xa2NextRollbackCount); 1943         assertEquals(1, xa3NextRollbackCount); 1944 1945         Thread.sleep(1000); // give time to cleanup for logger 1946 1947         cleanLogDir(dirName); 1948 1949         traceLog.info("DONE TEST"); 1950   } 1951    1952   public void testChecksumCorruptionAtMidLogFileArea() 1953      throws Exception 1954   { 1955      traceLog.info(">>> testChecksumCorruptionAtMidLogFileArea()"); 1956       1957      int n = 10; 1958      MockLogger logger = setupTxLogger(); 1959      TxManager tm = TxManager.getInstance(); 1960       1961      DummyXAResource xa1 = new DummyXAResourceImpl(); 1962      DummyXAResource xa2 = new DummyXAResourceImpl(); 1963      DummyXAResource xa3 = new DummyXAResourceImpl(); 1964 1965      DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 1966      DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 1967      DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 1968 1969      ArrayList recoverables = new ArrayList (); 1970      recoverables.add(rec1); 1971      recoverables.add(rec2); 1972      recoverables.add(rec3); 1973 1974      logger.setFailAfterCommitting(true); 1975      logger.setFailAfter(n); 1976       1977      try 1978      { 1979         for (int i = 0; ;i++) 1980         { 1981            tm.begin(); 1982            Transaction tx = tm.getTransaction(); 1983            tx.enlistResource(xa1); 1984            tx.enlistResource(xa2); 1985            tx.enlistResource(xa3); 1986            tm.commit(); 1987         } 1988      } 1989      catch (RecoveryTestingException e) 1990      { 1991         traceLog.info("Expected exception: " + e); 1992      } 1993      catch (Throwable e) 1994      { 1995         traceLog.info("Unexpected throwable:", e); 1996         fail("Unexpected throwable: " + e); 1997      } 1998 1999      tm.suspend(); // to disassociate thread with any txs. 2000 2001      logger.stop(); 2002 2003      int xa1Count = xa1.getCommittedCount(); 2004      int xa2Count = xa2.getCommittedCount(); 2005      int xa3Count = xa3.getCommittedCount(); 2006 2007      int xa1RollbackCount = xa1.getRollbackCount(); 2008      int xa2RollbackCount = xa2.getRollbackCount(); 2009      int xa3RollbackCount = xa3.getRollbackCount(); 2010 2011      try 2012      { 2013         logger = switchTxLogger(); 2014          2015         RecoveryLogReader[] readers = logger.getRecoveryLogs(); 2016         for (int i = 0; i < readers.length; i++) 2017         { 2018            File f = new File (readers[i].getLogFileName()); 2019            traceLog.info("log file: " + f.getName() + ", len: " + f.length()); 2020            if (f.length() == logFileSize) 2021            { 2022               RandomAccessFile raf = new RandomAccessFile (f, "rw"); 2023                2024               // Corrupt the last (the tenth) log record, by scanning 2025 // backwards and overwrite with a zero the last non-zero byte 2026 // in the log file. 2027 2028               for (long pos = logFileSize - 1; pos >= 0; pos--) 2029               { 2030                  raf.seek(pos); 2031                  byte b = raf.readByte(); 2032                  if (b != 0) 2033                  { 2034                     traceLog.info("Overwriting with a zero the last " + 2035                                   "non-zero byte in log file " + f.getName()); 2036                     raf.seek(pos); 2037                     raf.writeByte(0); 2038                     break; 2039                  } 2040               } 2041 2042               // Find the fourth log record. 2043 long pos = 0; 2044               for (int countMatches = 0; countMatches < 4; pos++) 2045               { 2046                  if (match(raf, pos, LogRecord.HEADER)) 2047                  { 2048                     traceLog.info("found log record at position " + pos); 2049                     countMatches++; 2050                  } 2051               } 2052               --pos; // point to first byte of fourth log record 2053 --pos; // point to last byte of third log record 2054 2055               // Now corrupt the third log record by scanning backwards and 2056 // overwriting with a zero the last non-zero byte that precedes 2057 // the fourth log record. 2058 2059               raf.seek(pos); 2060               byte b = raf.readByte(); 2061               while (b == 0) 2062               { 2063                  pos--; 2064                  raf.seek(pos); 2065                  b = raf.readByte(); 2066               } 2067               traceLog.info("Corrupting the third record in log file " + 2068                             f.getName() + ", pos=" + pos); 2069               raf.seek(pos); 2070               raf.writeByte(0); 2071                2072               // Also corrupt the next log record to check if the forward 2073 // scan can handle more than one bad records. 2074 ++pos; // point to first byte of fourth log record 2075 ++pos; // point to second byte of fourth log record 2076 while (!match(raf, pos, LogRecord.HEADER)) 2077                  pos++; 2078               // Now pos points to the first byte of the fifth log record. 2079 traceLog.info("found log record at position " + pos); 2080               --pos; // point to last byte of fourth log record 2081 2082               traceLog.info("Corrupting the fourth record in log file " + 2083                     f.getName() + ", pos=" + pos); 2084               raf.seek(pos); 2085               raf.writeByte(0); 2086            } 2087         } 2088          2089         RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 2090         manager.recover(recoverables); 2091         logger.stop(); 2092      } 2093      catch (CorruptedLogRecordException e) 2094      { 2095         traceLog.info("Expected exception:", e); 2096      } 2097      catch (Throwable e) 2098      { 2099         traceLog.info("Unexpected throwable:", e); 2100         fail("Unexpected throwable: " + e); 2101      } 2102 2103      int xa1NextCount = xa1.getCommittedCount(); 2104      int xa2NextCount = xa2.getCommittedCount(); 2105      int xa3NextCount = xa3.getCommittedCount(); 2106 2107      int xa1NextRollbackCount = xa1.getRollbackCount(); 2108      int xa2NextRollbackCount = xa2.getRollbackCount(); 2109      int xa3NextRollbackCount = xa3.getRollbackCount(); 2110 2111      int xa1PrepareCount = xa1.getPreparedCount(); 2112      int xa2PrepareCount = xa2.getPreparedCount(); 2113      int xa3PrepareCount = xa3.getPreparedCount(); 2114 2115      assertEquals(n, xa1Count); 2116      assertEquals(n, xa2Count); 2117      assertEquals(n, xa3Count); 2118      assertEquals(n, xa1NextCount); 2119      assertEquals(n, xa2NextCount); 2120      assertEquals(n, xa3NextCount); 2121 2122      assertEquals(0, xa1RollbackCount); 2123      assertEquals(0, xa2RollbackCount); 2124      assertEquals(0, xa3RollbackCount); 2125       2126      // In addition to the bad records area in the middle of the log file, 2127 // we also have a bad record at the very end of the log file, for a 2128 // transaction whose XA resources were still prepared at recovery time. 2129 // The log corruption in the middle of the log file disabled presumed 2130 // rollback, so the prepared resources should not have been rolledback. 2131 2132      assertEquals(0, xa1NextRollbackCount); // no rollbacks 2133 assertEquals(0, xa2NextRollbackCount); 2134      assertEquals(0, xa3NextRollbackCount); 2135 2136      assertEquals(1, xa1PrepareCount - xa1NextCount); // still prepared 2137 assertEquals(1, xa2PrepareCount - xa2NextCount); 2138      assertEquals(1, xa3PrepareCount - xa3NextCount); 2139       2140      Thread.sleep(1000); // give time to cleanup for logger 2141 2142      cleanLogDir(dirName); 2143 2144      traceLog.info("DONE TEST"); 2145   } 2146    2147   public void testHeaderCorruptionAtMidLogFileArea() 2148      throws Exception 2149   { 2150      traceLog.info(">>> testHeaderCorruptionAtMidLogFileArea()"); 2151       2152      int n = 10; 2153      MockLogger logger = setupTxLogger(); 2154      TxManager tm = TxManager.getInstance(); 2155       2156      DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 2157      DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 2158      DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 2159 2160      DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 2161      DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 2162      DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 2163 2164      ArrayList recoverables = new ArrayList (); 2165      recoverables.add(rec1); 2166      recoverables.add(rec2); 2167      recoverables.add(rec3); 2168 2169      logger.setFailAfterCommitting(true); 2170      logger.setFailAfter(n); 2171       2172      try 2173      { 2174         for (int i = 0; ;i++) 2175         { 2176            tm.begin(); 2177            Transaction tx = tm.getTransaction(); 2178            tx.enlistResource(xa1); 2179            tx.enlistResource(xa2); 2180            tx.enlistResource(xa3); 2181            tm.commit(); 2182         } 2183      } 2184      catch (RecoveryTestingException e) 2185      { 2186         traceLog.info("Expected exception: " + e); 2187      } 2188      catch (Throwable e) 2189      { 2190         traceLog.info("Unexpected throwable:", e); 2191         fail("Unexpected throwable: " + e); 2192      } 2193 2194      tm.suspend(); // to disassociate thread with any txs. 2195 2196      logger.stop(); 2197 2198      int xa1Count = xa1.getCommittedCount(); 2199      int xa2Count = xa2.getCommittedCount(); 2200      int xa3Count = xa3.getCommittedCount(); 2201 2202      int xa1RollbackCount = xa1.getRollbackCount(); 2203      int xa2RollbackCount = xa2.getRollbackCount(); 2204      int xa3RollbackCount = xa3.getRollbackCount(); 2205 2206      try 2207      { 2208         logger = switchTxLogger(); 2209          2210         RecoveryLogReader[] readers = logger.getRecoveryLogs(); 2211         for (int i = 0; i < readers.length; i++) 2212         { 2213            File f = new File (readers[i].getLogFileName()); 2214            traceLog.info("log file: " + f.getName() + ", len: " + f.length()); 2215            if (f.length() == logFileSize) 2216            { 2217               RandomAccessFile raf = new RandomAccessFile (f, "rw"); 2218                2219               // Corrupt the last (the tenth) log record, by scanning 2220 // backwards and overwrite with a zero the last non-zero byte 2221 // in the log file. 2222 2223               for (long pos = logFileSize - 1; pos >= 0; pos--) 2224               { 2225                  raf.seek(pos); 2226                  byte b = raf.readByte(); 2227                  if (b != 0) 2228                  { 2229                     traceLog.info("Overwriting with a zero the last " + 2230                                   "non-zero byte in log file " + f.getName()); 2231                     raf.seek(pos); 2232                     raf.writeByte(0); 2233                     break; 2234                  } 2235               } 2236 2237               // Find the third log record. 2238 long pos = 0; 2239               for (int countMatches = 0; countMatches < 3; pos++) 2240               { 2241                  if (match(raf, pos, LogRecord.HEADER)) 2242                  { 2243                     traceLog.info("found log record at position " + pos); 2244                     countMatches++; 2245                  } 2246               } 2247                2248               traceLog.info("Overwriting with a zero the second header " + 2249                             "byte of the third record in log file " + 2250                             f.getName()); 2251               raf.seek(pos); 2252               raf.writeByte(0); 2253                2254               // Also corrupt the next log record to check if the forward 2255 // scan can handle more than one bad records. 2256 while (!match(raf, pos, LogRecord.HEADER)) 2257                  pos++; 2258               // Now pos points to the first byte of the fourth log record. 2259 traceLog.info("found log record at position " + pos); 2260 2261               traceLog.info("Overwriting with a zero the first header " + 2262                             "byte of the fourth record in log file " + 2263                             f.getName()); 2264               raf.seek(pos); 2265               raf.writeByte(0); 2266            } 2267         } 2268          2269         RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 2270         manager.recover(recoverables); 2271         logger.stop(); 2272      } 2273      catch (CorruptedLogRecordException e) 2274      { 2275         traceLog.info("Expected exception:", e); 2276      } 2277      catch (Throwable e) 2278      { 2279         traceLog.info("Unexpected throwable:", e); 2280         fail("Unexpected throwable: " + e); 2281      } 2282 2283      int xa1NextCount = xa1.getCommittedCount(); 2284      int xa2NextCount = xa2.getCommittedCount(); 2285      int xa3NextCount = xa3.getCommittedCount(); 2286 2287      int xa1NextRollbackCount = xa1.getRollbackCount(); 2288      int xa2NextRollbackCount = xa2.getRollbackCount(); 2289      int xa3NextRollbackCount = xa3.getRollbackCount(); 2290 2291      int xa1PrepareCount = xa1.getPreparedCount(); 2292      int xa2PrepareCount = xa2.getPreparedCount(); 2293      int xa3PrepareCount = xa3.getPreparedCount(); 2294 2295      assertEquals(n, xa1Count); 2296      assertEquals(n, xa2Count); 2297      assertEquals(n, xa3Count); 2298      assertEquals(n, xa1NextCount); 2299      assertEquals(n, xa2NextCount); 2300      assertEquals(n, xa3NextCount); 2301 2302      assertEquals(0, xa1RollbackCount); 2303      assertEquals(0, xa2RollbackCount); 2304      assertEquals(0, xa3RollbackCount); 2305       2306      // In addition to the bad records area in the middle of the log file, 2307 // we also have a bad record at the very end of the log file, for a 2308 // transaction whose XA resources were still prepared at recovery time. 2309 // The log corruption in the middle of the log file disabled presumed 2310 // rollback, so the prepared resources should not have been rolledback. 2311 2312      assertEquals(0, xa1NextRollbackCount); // no rollbacks 2313 assertEquals(0, xa2NextRollbackCount); 2314      assertEquals(0, xa3NextRollbackCount); 2315 2316      assertEquals(1, xa1PrepareCount - xa1NextCount); // still prepared 2317 assertEquals(1, xa2PrepareCount - xa2NextCount); 2318      assertEquals(1, xa3PrepareCount - xa3NextCount); 2319       2320      Thread.sleep(1000); // give time to cleanup for logger 2321 2322      cleanLogDir(dirName); 2323 2324      traceLog.info("DONE TEST"); 2325   } 2326    2327   public void testRecLenCorruptionAtMidLogFileArea() 2328      throws Exception 2329   { 2330         traceLog.info(">>> testRecLenCorruptionAtMidLogFileArea()"); 2331          2332         int n = 10; 2333         MockLogger logger = setupTxLogger(); 2334         TxManager tm = TxManager.getInstance(); 2335          2336         DummyXAResourceImpl xa1 = new DummyXAResourceImpl(); 2337         DummyXAResourceImpl xa2 = new DummyXAResourceImpl(); 2338         DummyXAResourceImpl xa3 = new DummyXAResourceImpl(); 2339 2340         DummyRecoverable rec1 = new DummyRecoverable("one", xa1); 2341         DummyRecoverable rec2 = new DummyRecoverable("two", xa2); 2342         DummyRecoverable rec3 = new DummyRecoverable("three", xa3); 2343 2344         ArrayList recoverables = new ArrayList (); 2345         recoverables.add(rec1); 2346         recoverables.add(rec2); 2347         recoverables.add(rec3); 2348 2349         logger.setFailAfterCommitting(true); 2350         logger.setFailAfter(n); 2351          2352         try 2353         { 2354            for (int i = 0; ;i++) 2355            { 2356               tm.begin(); 2357               Transaction tx = tm.getTransaction(); 2358               tx.enlistResource(xa1); 2359               tx.enlistResource(xa2); 2360               tx.enlistResource(xa3); 2361               tm.commit(); 2362            } 2363         } 2364         catch (RecoveryTestingException e) 2365         { 2366            traceLog.info("Expected exception: " + e); 2367         } 2368         catch (Throwable e) 2369         { 2370            traceLog.info("Unexpected throwable:", e); 2371            fail("Unexpected throwable: " + e); 2372         } 2373 2374         tm.suspend(); // to disassociate thread with any txs. 2375 2376         logger.stop(); 2377 2378         int xa1Count = xa1.getCommittedCount(); 2379         int xa2Count = xa2.getCommittedCount(); 2380         int xa3Count = xa3.getCommittedCount(); 2381 2382         int xa1RollbackCount = xa1.getRollbackCount(); 2383         int xa2RollbackCount = xa2.getRollbackCount(); 2384         int xa3RollbackCount = xa3.getRollbackCount(); 2385 2386         try 2387         { 2388            logger = switchTxLogger(); 2389             2390            RecoveryLogReader[] readers = logger.getRecoveryLogs(); 2391            for (int i = 0; i < readers.length; i++) 2392            { 2393               File f = new File (readers[i].getLogFileName()); 2394               traceLog.info("log file: " + f.getName() + ", len: " + f.length()); 2395               if (f.length() == logFileSize) 2396               { 2397                  RandomAccessFile raf = new RandomAccessFile (f, "rw"); 2398                   2399                  // Corrupt the last (the tenth) log record, by scanning 2400 // backwards and overwrite with a zero the last non-zero byte 2401 // in the log file. 2402 2403                  for (long pos = logFileSize - 1; pos >= 0; pos--) 2404                  { 2405                     raf.seek(pos); 2406                     byte b = raf.readByte(); 2407                     if (b != 0) 2408                     { 2409                        traceLog.info("Overwriting with a zero the last " + 2410                                      "non-zero byte in log file " + f.getName()); 2411                        raf.seek(pos); 2412                        raf.writeByte(0); 2413                        break; 2414                     } 2415                  } 2416 2417                  // Find the third log record. 2418 long pos = 0; 2419                  for (int countMatches = 0; countMatches < 3; pos++) 2420                  { 2421                     if (match(raf, pos, LogRecord.HEADER)) 2422                     { 2423                        traceLog.info("found log record at position " + pos); 2424                        countMatches++; 2425                     } 2426                  } 2427                  pos += LogRecord.HEADER_LEN - 1; 2428                  traceLog.info("Changing the first recLen byte of the " + 2429                                "third record in log file " + f.getName()); 2430                  raf.seek(pos); 2431                  byte b = raf.readByte(); 2432                  raf.seek(pos); 2433                  raf.writeByte(b + 1); 2434                   2435                  // Also corrupt the next log record to check if the forward 2436 // scan can handle more than one bad records. 2437 while (!match(raf, pos, LogRecord.HEADER)) 2438                     pos++; 2439                  // Now pos points to the first byte of the fourth log record. 2440 traceLog.info("found log record at position " + pos); 2441 2442                  pos += LogRecord.HEADER_LEN; 2443                  traceLog.info("Changing the first recLen byte of the " + 2444                                "fourth record in log file " + f.getName()); 2445                  raf.seek(pos); 2446                  b = raf.readByte(); 2447                  raf.seek(pos); 2448                  raf.writeByte(b + 1); 2449               } 2450            } 2451             2452            RecoveryManager manager = new RecoveryManager(xidFactory, tm, logger); 2453            manager.recover(recoverables); 2454            logger.stop(); 2455         } 2456         catch (CorruptedLogRecordException e) 2457         { 2458            traceLog.info("Expected exception:", e); 2459         } 2460         catch (Throwable e) 2461         { 2462            traceLog.info("Unexpected throwable:", e); 2463            fail("Unexpected throwable: " + e); 2464         } 2465 2466         int xa1NextCount = xa1.getCommittedCount(); 2467         int xa2NextCount = xa2.getCommittedCount(); 2468         int xa3NextCount = xa3.getCommittedCount(); 2469 2470         int xa1NextRollbackCount = xa1.getRollbackCount(); 2471         int xa2NextRollbackCount = xa2.getRollbackCount(); 2472         int xa3NextRollbackCount = xa3.getRollbackCount(); 2473 2474         int xa1PrepareCount = xa1.getPreparedCount(); 2475         int xa2PrepareCount = xa2.getPreparedCount(); 2476         int xa3PrepareCount = xa3.getPreparedCount(); 2477 2478         assertEquals(n, xa1Count); 2479         assertEquals(n, xa2Count); 2480         assertEquals(n, xa3Count); 2481         assertEquals(n, xa1NextCount); 2482         assertEquals(n, xa2NextCount); 2483         assertEquals(n, xa3NextCount); 2484 2485         assertEquals(0, xa1RollbackCount); 2486         assertEquals(0, xa2RollbackCount); 2487         assertEquals(0, xa3RollbackCount); 2488          2489         // In addition to the bad records area in the middle of the log file, 2490 // we also have a bad record at the very end of the log file, for a 2491 // transaction whose XA resources were still prepared at recovery time. 2492 // The log corruption in the middle of the log file disabled presumed 2493 // rollback, so the prepared resources should not have been rolledback. 2494 2495         assertEquals(0, xa1NextRollbackCount); // no rollbacks 2496 assertEquals(0, xa2NextRollbackCount); 2497         assertEquals(0, xa3NextRollbackCount); 2498 2499         assertEquals(1, xa1PrepareCount - xa1NextCount); // still prepared 2500 assertEquals(1, xa2PrepareCount - xa2NextCount); 2501         assertEquals(1, xa3PrepareCount - xa3NextCount); 2502          2503         Thread.sleep(1000); // give time to cleanup for logger 2504 2505         cleanLogDir(dirName); 2506 2507         traceLog.info("DONE TEST"); 2508   } 2509    2510   private MockLogger setupTxLogger() throws Exception 2511   { 2512      cleanLogDir(dirName); 2513      return switchTxLogger(); 2514   } 2515    2516   private MockLogger switchTxLogger() throws Exception 2517   { 2518      MockLogger logger = new MockLogger(); 2519       2520      logger.setDirectoryList(directoryList); 2521      logger.setHeuristicStatusLogDirectory(heuristicDir); 2522      logger.setLogFileSize(logFileSize); 2523      logger.setXidFactory(xidFactory); 2524      logger.start(); 2525 2526      TxManager.getInstance().clear(); 2527      TxManager.getInstance().setRecoveryLogger(logger); 2528       2529      return logger; 2530   } 2531 2532   private void cleanLogDir(String dirName) 2533   { 2534      File dir = new File (dirName); 2535      dir = dir.getAbsoluteFile(); 2536      if (dir.exists()) 2537      { 2538         File [] logs = dir.listFiles(); 2539         if (logs != null) 2540         { 2541            for (int i = 0; i < logs.length; i++) 2542            { 2543               if (logs[i].exists()) logs[i].delete(); 2544            } 2545         } 2546      } 2547   } 2548 2549   /** 2550    * Returns true if the byte sequence that starts at given position of a 2551    * file matches a given byte array. 2552    * 2553    * @param file a random access file open for reading 2554    * @param pos the file position of the byte sequence to be compared against 2555    * the <code>pattern</code> byte array 2556    * @param pattern the byte pattern to match. 2557    * @return true if the pattern appears at the specified position of the 2558    * file, and false otherwise/ 2559    * @throws IOException if there is a problem reading the file. 2560    */ 2561   private static boolean match(RandomAccessFile file, long pos, byte[] pattern) 2562      throws IOException 2563   { 2564      for (int i = 0; i < pattern.length; i++) 2565      { 2566         file.seek(pos + i); 2567         if (file.readByte() != pattern[i]) 2568            return false; 2569      } 2570      return true; 2571   } 2572    2573 2574   public static Test suite() throws Exception 2575   { 2576      TestSuite suite = new TestSuite(); 2577      suite.addTest(new TestSuite(RecoverabilityTestCase.class)); 2578      return suite; 2579   } 2580} 2581

A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame Popular Tags