Java > Open Source Codes > com > sleepycat > je > cleaner > TruncateAndRemoveTest


1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002,2006 Oracle. All rights reserved.
5  *
6  * $Id: TruncateAndRemoveTest.java,v 1.17 2006/11/17 23:47:28 mark Exp $
7  */
8
9 package com.sleepycat.je.cleaner;
10
11 import java.io.File ;
12 import java.io.IOException ;
13 import java.util.HashSet ;
14 import java.util.Iterator ;
15 import java.util.Set ;
16 import java.nio.ByteBuffer ;
17
18 import junit.framework.TestCase;
19
20 import com.sleepycat.je.CheckpointConfig;
21 import com.sleepycat.je.Cursor;
22 import com.sleepycat.je.Database;
23 import com.sleepycat.je.DatabaseConfig;
24 import com.sleepycat.je.DatabaseEntry;
25 import com.sleepycat.je.DatabaseException;
26 import com.sleepycat.je.DbInternal;
27 import com.sleepycat.je.Environment;
28 import com.sleepycat.je.EnvironmentConfig;
29 import com.sleepycat.je.LockMode;
30 import com.sleepycat.je.OperationStatus;
31 import com.sleepycat.je.Transaction;
32 import com.sleepycat.je.config.EnvironmentParams;
33 import com.sleepycat.je.dbi.DatabaseId;
34 import com.sleepycat.je.dbi.DatabaseImpl;
35 import com.sleepycat.je.junit.JUnitThread;
36 import com.sleepycat.je.log.DumpFileReader;
37 import com.sleepycat.je.log.FileManager;
38 import com.sleepycat.je.log.LogEntryType;
39 import com.sleepycat.je.log.entry.INLogEntry;
40 import com.sleepycat.je.log.entry.LNLogEntry;
41 import com.sleepycat.je.log.entry.LogEntry;
42 import com.sleepycat.je.util.TestUtils;
43 import com.sleepycat.je.utilint.DbLsn;
44 import com.sleepycat.je.utilint.TestHook;
45
46 public class TruncateAndRemoveTest extends TestCase {
47
48     private static final String DB_NAME1 = "foo";
49     private static final String DB_NAME2 = "bar";
50     private static final long RECORD_COUNT = 100;
51
52     private static final CheckpointConfig FORCE_CHECKPOINT =
53         new CheckpointConfig();
54     static {
55         FORCE_CHECKPOINT.setForce(true);
56     }
57    
58     private static final boolean DEBUG = false;
59
60     private File envHome;
61     private Environment env;
62     private Database db;
63     private JUnitThread junitThread;
64     private boolean fetchObsoleteSize;
65
66     public TruncateAndRemoveTest() {
67         envHome = new File (System.getProperty(TestUtils.DEST_DIR));
68     }
69
70     public void setUp()
71         throws IOException , DatabaseException {
72
73         TestUtils.removeLogFiles("Setup", envHome, false);
74         TestUtils.removeFiles("Setup", envHome, FileManager.DEL_SUFFIX);
75     }
76
77     public void tearDown()
78         throws IOException , DatabaseException {
79   
80         if (junitThread != null) {
81             while (junitThread.isAlive()) {
82                 junitThread.interrupt();
83                 Thread.yield();
84             }
85             junitThread = null;
86         }
87
88         try {
89             if (env != null) {
90                 env.close();
91             }
92         } catch (Throwable e) {
93             System.out.println("tearDown: " + e);
94         }
95                 
96         try {
97             //*
98 TestUtils.removeLogFiles("tearDown", envHome, true);
99             TestUtils.removeFiles("tearDown", envHome, FileManager.DEL_SUFFIX);
100             //*/
101 } catch (Throwable e) {
102             System.out.println("tearDown: " + e);
103         }
104
105         db = null;
106         env = null;
107         envHome = null;
108     }
109
110     /**
111      * Opens the environment.
112      */
113     private void openEnv(boolean transactional)
114         throws DatabaseException {
115
116         EnvironmentConfig config = TestUtils.initEnvConfig();
117         config.setTransactional(transactional);
118         config.setAllowCreate(true);
119         /* Do not run the daemons since they interfere with LN counting. */
120         config.setConfigParam
121             (EnvironmentParams.ENV_RUN_CLEANER.getName(), "false");
122         config.setConfigParam
123             (EnvironmentParams.ENV_RUN_EVICTOR.getName(), "false");
124         config.setConfigParam
125         (EnvironmentParams.ENV_RUN_CHECKPOINTER.getName(), "false");
126         config.setConfigParam
127             (EnvironmentParams.ENV_RUN_INCOMPRESSOR.getName(), "false");
128
129         /* Use small nodes to test the post-txn scanning. */
130         config.setConfigParam
131             (EnvironmentParams.NODE_MAX.getName(), "10");
132         config.setConfigParam
133             (EnvironmentParams.NODE_MAX_DUPTREE.getName(), "10");
134
135         /* Use small files to ensure that there is cleaning. */
136         config.setConfigParam("je.cleaner.minUtilization", "90");
137         DbInternal.disableParameterValidation(config);
138         config.setConfigParam("je.log.fileMax", "4000");
139         
140         /* Obsolete LN size counting is optional per test. */
141         if (fetchObsoleteSize) {
142             config.setConfigParam
143                 (EnvironmentParams.CLEANER_FETCH_OBSOLETE_SIZE.getName(),
144                  "true");
145         }
146
147         env = new Environment(envHome, config);
148     }
149
150     /**
151      * Opens that database.
152      */
153     private void openDb(Transaction useTxn, String dbName)
154         throws DatabaseException {
155
156         DatabaseConfig dbConfig = new DatabaseConfig();
157         EnvironmentConfig envConfig = env.getConfig();
158         dbConfig.setTransactional(envConfig.getTransactional());
159         dbConfig.setAllowCreate(true);
160         db = env.openDatabase(useTxn, dbName, dbConfig);
161     }
162
163     /**
164      * Closes the environment and database.
165      */
166     private void closeEnv()
167         throws DatabaseException {
168
169         if (db != null) {
170             db.close();
171             db = null;
172         }
173         if (env != null) {
174             env.close();
175             env = null;
176         }
177     }
178
179     /**
180      * Test that truncate generates the right number of obsolete LNs.
181      */
182     public void testTruncate()
183         throws Exception {
184
185         openEnv(true);
186         openDb(null, DB_NAME1);
187         writeAndCountRecords(null, RECORD_COUNT);
188         DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId();
189         db.close();
190         db = null;
191
192         Transaction txn = env.beginTransaction(null, null);
193         truncate(txn, true);
194         ObsoleteCounts beforeCommit = getObsoleteCounts();
195         txn.commit();
196
197         verifyUtilization(beforeCommit,
198                           RECORD_COUNT + // LNs
199 3, // prev MapLN + deleted MapLN + prev NameLN
200 15); // 1 root, 2 INs, 12 BINs
201
202         closeEnv();
203         batchCleanAndVerify(saveId);
204     }
205
206     /**
207      * Test that aborting truncate generates the right number of obsolete LNs.
208      */
209     public void testTruncateAbort()
210         throws Exception {
211
212         openEnv(true);
213         openDb(null, DB_NAME1);
214         writeAndCountRecords(null, RECORD_COUNT);
215         db.close();
216         db = null;
217
218         Transaction txn = env.beginTransaction(null, null);
219         truncate(txn, true);
220         ObsoleteCounts beforeAbort = getObsoleteCounts();
221         txn.abort();
222
223         /*
224          * The obsolete count should include the records inserted after
225          * the truncate.
226          */
227         verifyUtilization(beforeAbort,
228                           /* 1 new nameLN, 2 copies of MapLN for new db */
229                            3,
230                            0);
231
232         /* Reopen, db should be populated. */
233         openDb(null, DB_NAME1);
234         assertEquals(RECORD_COUNT, countRecords(null));
235         closeEnv();
236     }
237
238     /**
239      * Test that aborting truncate generates the right number of obsolete LNs.
240      */
241     public void testTruncateRepopulateAbort()
242         throws Exception {
243
244         openEnv(true);
245         openDb(null, DB_NAME1);
246         writeAndCountRecords(null, RECORD_COUNT);
247         db.close();
248         db = null;
249
250         Transaction txn = env.beginTransaction(null, null);
251         truncate(txn, true);
252
253         /* populate the database with some more records. */
254         openDb(txn, DB_NAME1);
255         writeAndCountRecords(txn, RECORD_COUNT/4);
256         DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId();
257         db.close();
258         db = null;
259         ObsoleteCounts beforeAbort = getObsoleteCounts();
260         txn.abort();
261
262         /*
263          * The obsolete count should include the records inserted after
264          * the truncate.
265          */
266         verifyUtilization(beforeAbort,
267                           /* newly inserted LNs, 1 new nameLN,
268                            * 2 copies of MapLN for new db */
269                           (RECORD_COUNT/4) + 3,
270                           5);
271
272         /* Reopen, db should be populated. */
273         openDb(null, DB_NAME1);
274         assertEquals(RECORD_COUNT, countRecords(null));
275
276         closeEnv();
277         batchCleanAndVerify(saveId);
278     }
279
280     /**
281      * Test that remove generates the right number of obsolete LNs.
282      */
283     public void testRemove()
284         throws Exception {
285
286         openEnv(true);
287         openDb(null, DB_NAME1);
288         writeAndCountRecords(null, RECORD_COUNT);
289         DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId();
290         db.close();
291         db = null;
292
293         Transaction txn = env.beginTransaction(null, null);
294         env.removeDatabase(txn, DB_NAME1);
295         ObsoleteCounts beforeCommit = getObsoleteCounts();
296         txn.commit();
297
298         verifyUtilization(beforeCommit,
299                           /* LNs + old NameLN, old MapLN, delete MapLN */
300                           RECORD_COUNT + 3,
301                           15);
302
303         openDb(null, DB_NAME1);
304         assertEquals(0, countRecords(null));
305
306         closeEnv();
307         batchCleanAndVerify(saveId);
308     }
309
310     /**
311      * Test that remove generates the right number of obsolete LNs.
312      */
313     public void testNonTxnalRemove()
314         throws Exception {
315
316         openEnv(false);
317         openDb(null, DB_NAME1);
318         writeAndCountRecords(null, RECORD_COUNT);
319         DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId();
320         db.close();
321         db = null;
322         ObsoleteCounts beforeOperation = getObsoleteCounts();
323         env.removeDatabase(null, DB_NAME1);
324
325         verifyUtilization(beforeOperation,
326                           /* LNs + new NameLN, old NameLN, old MapLN, delete
327                              MapLN */
328                           RECORD_COUNT + 4,
329                           15);
330
331         openDb(null, DB_NAME1);
332         assertEquals(0, countRecords(null));
333
334         closeEnv();
335         batchCleanAndVerify(saveId);
336     }
337
338     /**
339      * Test that aborting remove generates the right number of obsolete LNs.
340      */
341     public void testRemoveAbort()
342         throws Exception {
343
344         /* Create database, populate, remove, abort the remove. */
345         openEnv(true);
346         openDb(null, DB_NAME1);
347         writeAndCountRecords(null, RECORD_COUNT);
348         db.close();
349         db = null;
350         Transaction txn = env.beginTransaction(null, null);
351         env.removeDatabase(txn, DB_NAME1);
352         ObsoleteCounts beforeAbort = getObsoleteCounts();
353         txn.abort();
354
355         verifyUtilization(beforeAbort, 0, 0);
356
357         /* All records should be there. */
358         openDb(null, DB_NAME1);
359         assertEquals(RECORD_COUNT, countRecords(null));
360
361         closeEnv();
362
363         /*
364          * Batch clean and then check the record count again, just to make sure
365          * we don't lose any valid data.
366          */
367         openEnv(true);
368         while (env.cleanLog() > 0) {
369         }
370         CheckpointConfig force = new CheckpointConfig();
371         force.setForce(true);
372         env.checkpoint(force);
373         closeEnv();
374
375         openEnv(true);
376         openDb(null, DB_NAME1);
377         assertEquals(RECORD_COUNT, countRecords(null));
378         closeEnv();
379     }
380
381     /**
382      * The same as testRemoveNotResident but forces fetching of obsolets LNs
383      * in order to count their sizes accurately.
384      */
385     public void testRemoveNotResidentFetchObsoleteSize()
386         throws Exception {
387
388         fetchObsoleteSize = true;
389         testRemoveNotResident();
390     }
391
392     /**
393      * Test that we can properly account for a non-resident database.
394      */
395     public void testRemoveNotResident()
396         throws Exception {
397
398         /* Create a database, populate. */
399         openEnv(true);
400         openDb(null, DB_NAME1);
401         writeAndCountRecords(null, RECORD_COUNT);
402         DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId();
403         db.close();
404         db = null;
405         env.close();
406         env = null;
407
408         /*
409          * Open the environment and remove the database. The
410          * database is not resident at all.
411          */
412         openEnv(true);
413         Transaction txn = env.beginTransaction(null, null);
414         env.removeDatabase(txn, DB_NAME1);
415         ObsoleteCounts beforeCommit = getObsoleteCounts();
416         txn.commit();
417
418         verifyUtilization(beforeCommit,
419                           /* LNs + old NameLN, old MapLN, delete MapLN */
420                           RECORD_COUNT + 3,
421                           /* 15 IN for data tree,
422                              2 for re-logged INs */
423                           15 + 2,
424                           true);
425
426         /* check record count. */
427         openDb(null, DB_NAME1);
428         assertEquals(0, countRecords(null));
429
430         closeEnv();
431         batchCleanAndVerify(saveId);
432     }
433
434     /**
435      * The same as testRemovePartialResident but forces fetching of obsolets
436      * LNs in order to count their sizes accurately.
437      */
438     public void testRemovePartialResidentFetchObsoleteSize()
439         throws Exception {
440
441         fetchObsoleteSize = true;
442         testRemovePartialResident();
443     }
444
445     /**
446      * Test that we can properly account for partially resident tree.
447      */
448     public void testRemovePartialResident()
449         throws Exception {
450
451         /* Create a database, populate. */
452         openEnv(true);
453         openDb(null, DB_NAME1);
454         writeAndCountRecords(null, RECORD_COUNT);
455         DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId();
456         db.close();
457         db = null;
458         env.close();
459         env = null;
460
461         /*
462          * Open the environment and remove the database. Pull 1 BIN in.
463          */
464         openEnv(true);
465         openDb(null, DB_NAME1);
466         Cursor c = db.openCursor(null, null);
467         assertEquals(OperationStatus.SUCCESS,
468                      c.getFirst(new DatabaseEntry(), new DatabaseEntry(),
469                                 LockMode.DEFAULT));
470         c.close();
471         db.close();
472         db = null;
473
474         Transaction txn = env.beginTransaction(null, null);
475         env.removeDatabase(txn, DB_NAME1);
476         ObsoleteCounts beforeCommit = getObsoleteCounts();
477         txn.commit();
478
479         verifyUtilization(beforeCommit,
480                           /* LNs + old NameLN, old MapLN, delete MapLN */
481                           RECORD_COUNT + 3,
482                           /* 15 IN for data tree, 2 for file summary db */
483                           15 + 2,
484                           true);
485
486         /* check record count. */
487         openDb(null, DB_NAME1);
488         assertEquals(0, countRecords(null));
489
490         closeEnv();
491         batchCleanAndVerify(saveId);
492     }
493   
494     /**
495      * Tests that a log file is not deleted by the cleaner when it contains
496      * entries in a database that is pending deletion.
497      */
498     public void testDBPendingDeletion()
499         throws DatabaseException, InterruptedException {
500
501         doDBPendingTest(RECORD_COUNT, false /*deleteAll*/, 7);
502     }
503
504     /**
505      * Like testDBPendingDeletion but creates a scenario where only a single
506      * log file is cleaned, and that log file contains only known obsolete
507      * log entries. This reproduced a bug where we neglected to adding
508      * pending deleted DBs to the cleaner's pending DB set if all entries in
509      * the log file were known obsoleted. [#13333]
510      */
511     public void testObsoleteLogFile()
512         throws DatabaseException, InterruptedException {
513
514         doDBPendingTest(40, true /*deleteAll*/, 1);
515     }
516     
517     private void doDBPendingTest(long recordCount,
518                                  boolean deleteAll,
519                                  int expectFilesCleaned)
520         throws DatabaseException, InterruptedException {
521
522         /* Create a database, populate, close. */
523         Set logFiles = new HashSet ();
524         openEnv(true);
525         openDb(null, DB_NAME1);
526         writeAndMakeWaste(recordCount, logFiles, deleteAll);
527         long remainingRecordCount = deleteAll ? 0 : recordCount;
528         env.checkpoint(FORCE_CHECKPOINT);
529         ObsoleteCounts obsoleteCounts = getObsoleteCounts();
530         DatabaseImpl dbImpl = DbInternal.dbGetDatabaseImpl(db);
531         db.close();
532         db = null;
533         assertTrue(!dbImpl.isDeleteFinished());
534         assertTrue(!dbImpl.isDeleted());
535
536         /* Make sure that we wrote a full file's worth of LNs. */
537         assertTrue(logFiles.size() >= 3);
538         assertTrue(logFilesExist(logFiles));
539
540         /* Remove the database but do not commit yet. */
541         final Transaction txn = env.beginTransaction(null, null);
542         env.removeDatabase(txn, DB_NAME1);
543
544         /* The obsolete count should be <= 1 (for the NameLN). */
545         obsoleteCounts = verifyUtilization(obsoleteCounts, 1, 0);
546
547         junitThread = new JUnitThread("Committer") {
548             public void testBody()
549                 throws DatabaseException {
550                 try {
551                     txn.commit();
552                 } catch (Throwable e) {
553                     e.printStackTrace();
554                 }
555             }
556         };
557
558         /*
559          * Set a hook to cause the commit to block. The commit is done in a
560          * separate thread. The commit will set the DB state to pendingDeleted
561          * and will then wait for the hook to return.
562          */
563         final Object lock = new Object ();
564
565         dbImpl.setPendingDeletedHook(new TestHook() {
566             public void doIOHook()
567                 throws IOException {
568                 throw new UnsupportedOperationException ();
569             }
570             public void doHook() {
571                 synchronized (lock) {
572                     try {
573                         lock.notify();
574                         lock.wait();
575                     } catch (InterruptedException e) {
576                         e.printStackTrace();
577                         throw new RuntimeException (e.toString());
578                     }
579                 }
580             }
581             public Object getHookValue() {
582             return null;
583             }
584         });
585
586         /* Start the committer thread; expect the pending deleted state. */
587         synchronized (lock) {
588             junitThread.start();
589             lock.wait();
590         }
591         assertTrue(!dbImpl.isDeleteFinished());
592         assertTrue(dbImpl.isDeleted());
593
594         /* Expect obsolete LNs: NameLN */
595         obsoleteCounts = verifyUtilization(obsoleteCounts, 1, 0);
596
597         /* The DB deletion is pending; the log file should still exist. */
598         int filesCleaned = env.cleanLog();
599         assertEquals(expectFilesCleaned, filesCleaned);
600         assertTrue(filesCleaned > 0);
601         env.checkpoint(FORCE_CHECKPOINT);
602         env.checkpoint(FORCE_CHECKPOINT);
603         assertTrue(logFilesExist(logFiles));
604
605         /*
606          * When the commiter thread finishes, the DB deletion will be
607          * complete and the DB state will change to deleted.
608          */
609         synchronized (lock) {
610             lock.notify();
611         }
612         try {
613             junitThread.finishTest();
614             junitThread = null;
615         } catch (Throwable e) {
616             e.printStackTrace();
617             fail(e.toString());
618         }
619         assertTrue(dbImpl.isDeleteFinished());
620         assertTrue(dbImpl.isDeleted());
621
622         /* Expect obsolete LNs: recordCount + MapLN + FSLNs (apprx). */
623         verifyUtilization(obsoleteCounts, remainingRecordCount + 6, 0);
624
625         /* The DB deletion is complete; the log file should be deleted. */
626         env.checkpoint(FORCE_CHECKPOINT);
627         env.checkpoint(FORCE_CHECKPOINT);
628         assertTrue(!logFilesExist(logFiles));
629     }
630
631     private void writeAndCountRecords(Transaction txn, long count)
632         throws DatabaseException {
633
634         for (int i = 1; i <= count; i += 1) {
635             DatabaseEntry entry = new DatabaseEntry(TestUtils.getTestArray(i));
636
637             db.put(txn, entry, entry);
638         }
639
640         /* Insert and delete some records, insert and abort some records. */
641         DatabaseEntry entry =
642             new DatabaseEntry(TestUtils.getTestArray((int)count+1));
643         db.put(txn, entry, entry);
644         db.delete(txn, entry);
645
646         EnvironmentConfig envConfig = env.getConfig();
647         if (envConfig.getTransactional()) {
648             entry = new DatabaseEntry(TestUtils.getTestArray(0));
649             Transaction txn2 = env.beginTransaction(null, null);
650             db.put(txn2, entry, entry);
651             txn2.abort();
652             txn2 = null;
653         }
654
655         assertEquals(count, countRecords(txn));
656     }
657
658     /**
659      * Writes the specified number of records to db. Check the number of
660      * records, and return the number of obsolete records. Returns a set of
661      * the file numbers that are written to.
662      *
663      * Makes waste (obsolete records): If doDelete=true, deletes records as
664      * they are added; otherwise does updates to produce obsolete records
665      * interleaved with non-obsolete records.
666      */
667     private void writeAndMakeWaste(long count,
668                                    Set logFilesWritten,
669                                    boolean doDelete)
670         throws DatabaseException {
671
672         Transaction txn = env.beginTransaction(null, null);
673         Cursor cursor = db.openCursor(txn, null);
674         for (int i = 0; i < count; i += 1) {
675             DatabaseEntry entry = new DatabaseEntry(TestUtils.getTestArray(i));
676             cursor.put(entry, entry);
677             /* Add log file written. */
678             long file = CleanerTestUtils.getLogFile(this, cursor);
679             logFilesWritten.add(new Long (file));
680             /* Make waste. */
681             if (!doDelete) {
682                 cursor.put(entry, entry);
683                 cursor.put(entry, entry);
684             }
685         }
686         if (doDelete) {
687             DatabaseEntry key = new DatabaseEntry();
688             DatabaseEntry data = new DatabaseEntry();
689             OperationStatus status;
690             for (status = cursor.getFirst(key, data, null);
691                  status == OperationStatus.SUCCESS;
692                  status = cursor.getNext(key, data, null)) {
693                 /* Make waste. */
694                 cursor.delete();
695                 /* Add log file written. */
696                 long file = CleanerTestUtils.getLogFile(this, cursor);
697                 logFilesWritten.add(new Long (file));
698             }
699         }
700         cursor.close();
701         txn.commit();
702         assertEquals(doDelete ? 0 : count, countRecords(null));
703     }
704
705     /* Truncate database and check the count. */
706     private void truncate(Transaction useTxn,
707                           boolean getCount)
708         throws DatabaseException {
709
710         long nTruncated = env.truncateDatabase(useTxn, DB_NAME1, getCount);
711
712         if (getCount) {
713             assertEquals(RECORD_COUNT, nTruncated);
714         }
715
716         assertEquals(0, countRecords(useTxn));
717     }
718                           
719     /**
720      * Returns how many records are in the database.
721      */
722     private int countRecords(Transaction useTxn)
723         throws DatabaseException {
724
725         DatabaseEntry key = new DatabaseEntry();
726         DatabaseEntry data = new DatabaseEntry();
727         if (db == null) {
728             openDb(useTxn, DB_NAME1);
729         }
730         Cursor cursor = db.openCursor(useTxn, null);
731         try {
732             int count = 0;
733             OperationStatus status = cursor.getFirst(key, data, null);
734             while (status == OperationStatus.SUCCESS) {
735                 count += 1;
736                 status = cursor.getNext(key, data, null);
737             }
738             return count;
739         } finally {
740             cursor.close();
741         }
742     }
743
744     /**
745      * Return the total number of obsolete node counts according to the
746      * UtilizationProfile and UtilizationTracker.
747      */
748     private ObsoleteCounts getObsoleteCounts()
749         throws DatabaseException {
750
751         FileSummary[] files = (FileSummary[])
752             DbInternal.envGetEnvironmentImpl(env)
753                       .getUtilizationProfile()
754                       .getFileSummaryMap(true)
755                       .values().toArray(new FileSummary[0]);
756         int lnCount = 0;
757         int inCount = 0;
758         int lnSize = 0;
759         int lnSizeCounted = 0;
760         for (int i = 0; i < files.length; i += 1) {
761             lnCount += files[i].obsoleteLNCount;
762             inCount += files[i].obsoleteINCount;
763             lnSize += files[i].obsoleteLNSize;
764             lnSizeCounted += files[i].obsoleteLNSizeCounted;
765         }
766
767         return new ObsoleteCounts(lnCount, inCount, lnSize, lnSizeCounted);
768     }
769
770     private class ObsoleteCounts {
771         int obsoleteLNs;
772         int obsoleteINs;
773         int obsoleteLNSize;
774         int obsoleteLNSizeCounted;
775
776         ObsoleteCounts(int obsoleteLNs,
777                        int obsoleteINs,
778                        int obsoleteLNSize,
779                        int obsoleteLNSizeCounted) {
780             this.obsoleteLNs = obsoleteLNs;
781             this.obsoleteINs = obsoleteINs;
782             this.obsoleteLNSize = obsoleteLNSize;
783             this.obsoleteLNSizeCounted = obsoleteLNSizeCounted;
784         }
785
786         public String toString() {
787             return "lns=" + obsoleteLNs + " ins=" + obsoleteINs +
788                    " lnSize=" + obsoleteLNSize +
789                    " lnSizeCounted=" + obsoleteLNSizeCounted;
790         }
791     }
792
793     private ObsoleteCounts verifyUtilization(ObsoleteCounts prev,
794                                              long expectedLNs,
795                                              int expectedINs)
796         throws DatabaseException {
797         
798         return verifyUtilization(prev, expectedLNs, expectedINs, false);
799     }
800
801     /*
802      * Check obsolete counts. If the expected IN count is zero, don't
803      * check the obsolete IN count. Always check the obsolete LN count.
804      */
805     private ObsoleteCounts verifyUtilization(ObsoleteCounts prev,
806                                              long expectedLNs,
807                                              int expectedINs,
808                                              boolean expectNonResident)
809         throws DatabaseException {
810
811         /*
812          * If all nodes are resident OR we have explicitly configured
813          * fetchObsoleteSize, then the size of every LN should have been
814          * counted.
815          */
816         boolean expectAccurateObsoleteLNSize =
817             !expectNonResident || fetchObsoleteSize;
818     
819         ObsoleteCounts now = getObsoleteCounts();
820         String beforeAndAfter = "before: " + prev + " now: " + now;
821         if (DEBUG) {
822             System.out.println(beforeAndAfter);
823         }
824
825         assertEquals(beforeAndAfter, expectedLNs,
826                      now.obsoleteLNs - prev.obsoleteLNs);
827         if (expectedLNs > 0) {
828             int size = now.obsoleteLNSize - prev.obsoleteLNSize;
829             int counted = now.obsoleteLNSizeCounted -
830                           prev.obsoleteLNSizeCounted;
831             assertTrue(String.valueOf(size), size > 0);
832
833             if (expectAccurateObsoleteLNSize) {
834                 assertEquals(beforeAndAfter, counted,
835                              now.obsoleteLNs - prev.obsoleteLNs);
836             }
837         }
838         if (expectedINs > 0) {
839             assertEquals(beforeAndAfter, expectedINs,
840                          now.obsoleteINs - prev.obsoleteINs);
841         }
842
843         /* Verify utilization using UtilizationFileReader. */
844         CleanerTestUtils.verifyUtilization
845             (DbInternal.envGetEnvironmentImpl(env),
846              true, // expectAccurateObsoleteLNCount
847 expectAccurateObsoleteLNSize);
848
849         return now;
850     }
851
852     /**
853      * Returns true if all files exist, or false if any file is deleted.
854      */
855     private boolean logFilesExist(Set fileNumbers) {
856
857         Iterator iter = fileNumbers.iterator();
858         while (iter.hasNext()) {
859             long fileNum = ((Long ) iter.next()).longValue();
860             File file = new File
861                 (envHome,
862                  FileManager.getFileName(fileNum, FileManager.JE_SUFFIX));
863             if (!file.exists()) {
864                 return false;
865             }
866         }
867         return true;
868     }
869
870     /*
871      * Run batch cleaning and verify that there are no files with these
872      * log entries.
873      */
874     private void batchCleanAndVerify(DatabaseId dbId)
875         throws Exception {
876
877         /*
878          * Open the environment, flip the log files to reduce mixing of new
879          * records and old records and add more records to force the
880          * utilization level of the removed records down.
881          */
882         openEnv(true);
883         openDb(null, DB_NAME2);
884         long lsn = DbInternal.envGetEnvironmentImpl(env).forceLogFileFlip();
885         CheckpointConfig force = new CheckpointConfig();
886         force.setForce(true);
887         env.checkpoint(force);
888
889         writeAndCountRecords(null, RECORD_COUNT * 3);
890         env.checkpoint(force);
891
892         db.close();
893         db = null;
894
895         /* Check log files, there should be entries with this database. */
896         CheckReader checker = new CheckReader(env, dbId, true);
897         while (checker.readNextEntry()) {
898         }
899
900         if (DEBUG) {
901             System.out.println("entries for this db =" + checker.getCount());
902         }
903
904         assertTrue(checker.getCount() > 0);
905
906         /* batch clean. */
907         boolean anyCleaned = false;
908         while (env.cleanLog() > 0) {
909             anyCleaned = true;
910         }
911
912         assertTrue(anyCleaned);
913
914         if (anyCleaned) {
915             env.checkpoint(force);
916         }
917
918         /* Check log files, there should be no entries with this database. */
919         checker = new CheckReader(env, dbId, false);
920         while (checker.readNextEntry()) {
921         }
922
923         closeEnv();
924         
925     }
926
927     class CheckReader extends DumpFileReader{
928
929         private DatabaseId dbId;
930         private boolean expectEntries;
931         private int count;
932
933         /*
934          * @param databaseId we're looking for log entries for this database.
935          * @param expectEntries if false, there should be no log entries
936          * with this database id. If true, the log should have entries
937          * with this database id.
938          */
939         CheckReader(Environment env, DatabaseId dbId, boolean expectEntries)
940             throws DatabaseException, IOException {
941             
942             super(DbInternal.envGetEnvironmentImpl(env),
943               1000, DbLsn.NULL_LSN, DbLsn.NULL_LSN,
944                   null, null, false);
945             this.dbId = dbId;
946             this.expectEntries = expectEntries;
947         }
948         
949         protected boolean processEntry(ByteBuffer entryBuffer)
950             throws DatabaseException {
951             
952             /* Figure out what kind of log entry this is */
953             LogEntryType lastEntryType =
954                 LogEntryType.findType(currentEntryTypeNum,
955                                       currentEntryTypeVersion);
956             boolean isNode = LogEntryType.isNodeType(currentEntryTypeNum,
957                                                      currentEntryTypeVersion);
958
959
960             /* Read the entry. */
961             LogEntry entry = lastEntryType.getSharedLogEntry();
962             entry.readEntry(entryBuffer, currentEntrySize,
963                             currentEntryTypeVersion, true);
964
965             long lsn = getLastLsn();
966             if (isNode) {
967                 boolean found = false;
968                 if (entry instanceof INLogEntry) {
969                     INLogEntry inEntry = (INLogEntry) entry;
970                     found = dbId.equals(inEntry.getDbId());
971                 } else {
972                     LNLogEntry lnEntry = (LNLogEntry) entry;
973                     found = dbId.equals(lnEntry.getDbId());
974                 }
975                 if (found) {
976                     if (expectEntries) {
977                         count++;
978                     } else {
979                         StringBuffer sb = new StringBuffer ();
980                         entry.dumpEntry(sb, false);
981                         fail("lsn=" + DbLsn.getNoFormatString(lsn) +
982                              " dbId = " + dbId +
983                              " entry= " + sb.toString());
984                     }
985                 }
986             }
987
988
989             return true;
990         }
991
992         /* Num entries with this database id seen by reader. */
993         int getCount() {
994             return count;
995         }
996     }
997 }
998

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags