KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > com > sleepycat > je > util > DbScavengerTest


1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002,2006 Oracle. All rights reserved.
5  *
6  * $Id: DbScavengerTest.java,v 1.16 2006/10/30 21:14:53 bostic Exp $
7  */

8
9 package com.sleepycat.je.util;
10
11 import java.io.BufferedReader JavaDoc;
12 import java.io.File JavaDoc;
13 import java.io.FileInputStream JavaDoc;
14 import java.io.FilenameFilter JavaDoc;
15 import java.io.IOException JavaDoc;
16 import java.io.InputStreamReader JavaDoc;
17 import java.io.RandomAccessFile JavaDoc;
18 import java.util.HashMap JavaDoc;
19 import java.util.HashSet JavaDoc;
20 import java.util.Iterator JavaDoc;
21 import java.util.Map JavaDoc;
22 import java.util.Set JavaDoc;
23 import java.util.StringTokenizer JavaDoc;
24
25 import junit.framework.TestCase;
26
27 import com.sleepycat.bind.tuple.IntegerBinding;
28 import com.sleepycat.je.Cursor;
29 import com.sleepycat.je.Database;
30 import com.sleepycat.je.DatabaseConfig;
31 import com.sleepycat.je.DatabaseEntry;
32 import com.sleepycat.je.DatabaseException;
33 import com.sleepycat.je.DatabaseNotFoundException;
34 import com.sleepycat.je.DbInternal;
35 import com.sleepycat.je.Environment;
36 import com.sleepycat.je.EnvironmentConfig;
37 import com.sleepycat.je.OperationStatus;
38 import com.sleepycat.je.Transaction;
39 import com.sleepycat.je.config.EnvironmentParams;
40 import com.sleepycat.je.log.FileManager;
41 import com.sleepycat.je.utilint.DbLsn;
42 import com.sleepycat.je.utilint.DbScavenger;
43
44 public class DbScavengerTest extends TestCase {
45
46     private static final int TRANSACTIONAL = 1 << 0;
47     private static final int WRITE_MULTIPLE = 1 << 1;
48     private static final int PRINTABLE = 1 << 2;
49     private static final int ABORT_BEFORE = 1 << 3;
50     private static final int ABORT_AFTER = 1 << 4;
51     private static final int CORRUPT_LOG = 1 << 5;
52     private static final int DELETE_DATA = 1 << 6;
53     private static final int AGGRESSIVE = 1 << 7;
54
55     private static final int N_DBS = 3;
56     private static final int N_KEYS = 100;
57     private static final int N_DATA_BYTES = 100;
58     private static final int LOG_SIZE = 10000;
59
60     private String JavaDoc envHomeName;
61     private File JavaDoc envHome;
62     
63     private Environment env;
64
65     private Database[] dbs = new Database[N_DBS];
66
67     private boolean duplicatesAllowed = true;
68
69     public DbScavengerTest() {
70     envHomeName = System.getProperty(TestUtils.DEST_DIR);
71         envHome = new File JavaDoc(envHomeName);
72     }
73
74     public void setUp()
75     throws IOException JavaDoc {
76
77         TestUtils.removeLogFiles("Setup", envHome, false);
78         TestUtils.removeFiles("Setup", envHome, ".dump");
79     }
80     
81     public void tearDown()
82     throws IOException JavaDoc {
83
84         if (env != null) {
85             try {
86                 env.close();
87             } catch (Exception JavaDoc e) {
88                 System.out.println("TearDown: " + e);
89             }
90             env = null;
91         }
92         TestUtils.removeLogFiles("TearDown", envHome, false);
93         TestUtils.removeFiles("Teardown", envHome, ".dump");
94     }
95
96     public void testScavenger1()
97         throws Throwable JavaDoc {
98
99     try {
100         doScavengerTest(PRINTABLE | TRANSACTIONAL |
101                 ABORT_BEFORE | ABORT_AFTER);
102     } catch (Throwable JavaDoc T) {
103         System.out.println("caught " + T);
104         T.printStackTrace();
105     }
106     }
107
108     public void testScavenger2()
109         throws Throwable JavaDoc {
110
111     try {
112         doScavengerTest(PRINTABLE | TRANSACTIONAL | ABORT_BEFORE);
113     } catch (Throwable JavaDoc T) {
114         System.out.println("caught " + T);
115         T.printStackTrace();
116     }
117     }
118
119     public void testScavenger3()
120         throws Throwable JavaDoc {
121
122     try {
123         doScavengerTest(PRINTABLE | TRANSACTIONAL | ABORT_AFTER);
124     } catch (Throwable JavaDoc T) {
125         System.out.println("caught " + T);
126         T.printStackTrace();
127     }
128     }
129
130     public void testScavenger4()
131         throws Throwable JavaDoc {
132
133     try {
134         doScavengerTest(PRINTABLE | TRANSACTIONAL);
135     } catch (Throwable JavaDoc T) {
136         System.out.println("caught " + T);
137         T.printStackTrace();
138     }
139     }
140
141     public void testScavenger5()
142         throws Throwable JavaDoc {
143
144     try {
145         doScavengerTest(PRINTABLE | WRITE_MULTIPLE | TRANSACTIONAL);
146     } catch (Throwable JavaDoc T) {
147         System.out.println("caught " + T);
148         T.printStackTrace();
149     }
150     }
151
152     public void testScavenger6()
153         throws Throwable JavaDoc {
154
155     try {
156         doScavengerTest(PRINTABLE);
157     } catch (Throwable JavaDoc T) {
158         System.out.println("caught " + T);
159         T.printStackTrace();
160         throw T;
161     }
162     }
163
164     public void testScavenger7()
165         throws Throwable JavaDoc {
166
167     try {
168         doScavengerTest(TRANSACTIONAL | ABORT_BEFORE | ABORT_AFTER);
169     } catch (Throwable JavaDoc T) {
170         System.out.println("caught " + T);
171         T.printStackTrace();
172     }
173     }
174
175     public void testScavenger8()
176         throws Throwable JavaDoc {
177
178     try {
179         doScavengerTest(TRANSACTIONAL | ABORT_BEFORE);
180     } catch (Throwable JavaDoc T) {
181         System.out.println("caught " + T);
182         T.printStackTrace();
183     }
184     }
185
186     public void testScavenger9()
187         throws Throwable JavaDoc {
188
189     try {
190         doScavengerTest(TRANSACTIONAL);
191     } catch (Throwable JavaDoc T) {
192         System.out.println("caught " + T);
193         T.printStackTrace();
194     }
195     }
196
197     public void testScavenger10()
198         throws Throwable JavaDoc {
199
200     try {
201         doScavengerTest(TRANSACTIONAL | ABORT_AFTER);
202     } catch (Throwable JavaDoc T) {
203         System.out.println("caught " + T);
204         T.printStackTrace();
205     }
206     }
207
208     public void testScavenger11()
209         throws Throwable JavaDoc {
210
211     try {
212         doScavengerTest(0);
213     } catch (Throwable JavaDoc T) {
214         System.out.println("caught " + T);
215         T.printStackTrace();
216     }
217     }
218
219     public void testScavenger12()
220         throws Throwable JavaDoc {
221
222     try {
223         doScavengerTest(CORRUPT_LOG);
224     } catch (Throwable JavaDoc T) {
225         System.out.println("caught " + T);
226         T.printStackTrace();
227     }
228     }
229
230     public void testScavenger13()
231         throws Throwable JavaDoc {
232
233     try {
234         doScavengerTest(DELETE_DATA);
235     } catch (Throwable JavaDoc T) {
236         System.out.println("caught " + T);
237         T.printStackTrace();
238     }
239     }
240
241     public void testScavenger14()
242         throws Throwable JavaDoc {
243
244     try {
245         doScavengerTest(AGGRESSIVE);
246     } catch (Throwable JavaDoc T) {
247         System.out.println("caught " + T);
248         T.printStackTrace();
249     }
250     }
251
252     public void testScavengerAbortedDbLevelOperations()
253         throws Throwable JavaDoc {
254
255     try {
256         createEnv(true, true);
257         boolean doAbort = true;
258         byte[] dataBytes = new byte[N_DATA_BYTES];
259         DatabaseEntry key = new DatabaseEntry();
260         DatabaseEntry data = new DatabaseEntry(dataBytes);
261         IntegerBinding.intToEntry(1, key);
262         TestUtils.generateRandomAlphaBytes(dataBytes);
263         for (int i = 0; i < 2; i++) {
264         Transaction txn = env.beginTransaction(null, null);
265         for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
266             String JavaDoc databaseName = null;
267             if (doAbort) {
268             databaseName = "abortedDb" + dbCnt;
269             } else {
270             databaseName = "simpleDb" + dbCnt;
271             }
272             DatabaseConfig dbConfig = new DatabaseConfig();
273             dbConfig.setAllowCreate(true);
274             dbConfig.setSortedDuplicates(duplicatesAllowed);
275             dbConfig.setTransactional(true);
276             if (dbs[dbCnt] != null) {
277             throw new DatabaseException("database already open");
278             }
279             Database db =
280             env.openDatabase(txn, databaseName, dbConfig);
281             dbs[dbCnt] = db;
282             db.put(txn, key, data);
283         }
284         if (doAbort) {
285             txn.abort();
286             dbs = new Database[N_DBS];
287         } else {
288             txn.commit();
289         }
290         doAbort = !doAbort;
291         }
292
293         closeEnv();
294         createEnv(false, false);
295         openDbs(false, false, duplicatesAllowed, null);
296         dumpDbs(false, false);
297
298         /* Close the environment, delete it completely from the disk. */
299         closeEnv();
300         TestUtils.removeLogFiles("doScavengerTest", envHome, false);
301
302         /* Recreate and reload the environment from the scavenger files. */
303         createEnv(true, true);
304         loadDbs();
305
306         /* Verify that the data is the same as when it was created. */
307         for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
308         String JavaDoc databaseName = "abortedDb" + dbCnt;
309         DatabaseConfig dbConfig = new DatabaseConfig();
310         dbConfig.setAllowCreate(false);
311         try {
312             env.openDatabase(null, databaseName, dbConfig);
313             fail("expected DatabaseNotFoundException");
314         } catch (DatabaseNotFoundException DNFE) {
315             /* Expected. */
316         }
317         }
318         closeEnv();
319
320     } catch (Throwable JavaDoc T) {
321         System.out.println("caught " + T);
322         T.printStackTrace();
323     }
324     }
325
326     private void doScavengerTest(int config)
327     throws DatabaseException, IOException JavaDoc {
328
329     boolean printable = (config & PRINTABLE) != 0;
330     boolean transactional = (config & TRANSACTIONAL) != 0;
331     boolean writeMultiple = (config & WRITE_MULTIPLE) != 0;
332     boolean abortBefore = (config & ABORT_BEFORE) != 0;
333     boolean abortAfter = (config & ABORT_AFTER) != 0;
334     boolean corruptLog = (config & CORRUPT_LOG) != 0;
335     boolean deleteData = (config & DELETE_DATA) != 0;
336     boolean aggressive = (config & AGGRESSIVE) != 0;
337
338     assert transactional ||
339         (!abortBefore && !abortAfter);
340
341     Map JavaDoc[] dataMaps = new Map JavaDoc[N_DBS];
342     Set JavaDoc lsnsToCorrupt = new HashSet JavaDoc();
343     /* Create the environment and some data. */
344     createEnvAndDbs(dataMaps,
345             writeMultiple,
346             transactional,
347             abortBefore,
348             abortAfter,
349             corruptLog,
350             lsnsToCorrupt,
351             deleteData);
352     closeEnv();
353     createEnv(false, false);
354     if (corruptLog) {
355         corruptFiles(lsnsToCorrupt);
356     }
357     openDbs(false, false, duplicatesAllowed, null);
358     dumpDbs(printable, aggressive);
359
360     /* Close the environment, delete it completely from the disk. */
361     closeEnv();
362         TestUtils.removeLogFiles("doScavengerTest", envHome, false);
363
364     /* Recreate the environment and load it from the scavenger files. */
365     createEnv(true, transactional);
366     loadDbs();
367
368     /* Verify that the data is the same as when it was created. */
369     openDbs(false, false, duplicatesAllowed, null);
370     verifyDbs(dataMaps);
371     closeEnv();
372     }
373
374     private void closeEnv()
375     throws DatabaseException {
376
377     for (int i = 0; i < N_DBS; i++) {
378         if (dbs[i] != null) {
379         dbs[i].close();
380         dbs[i] = null;
381         }
382     }
383
384     env.close();
385         env = null;
386     }
387
388     private void createEnv(boolean create, boolean transactional)
389     throws DatabaseException {
390
391     EnvironmentConfig envConfig = TestUtils.initEnvConfig();
392     DbInternal.disableParameterValidation(envConfig);
393         envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(),
394                  "false");
395         envConfig.setConfigParam(EnvironmentParams.LOG_FILE_MAX.getName(),
396                  "" + LOG_SIZE);
397     envConfig.setTransactional(transactional);
398     envConfig.setAllowCreate(create);
399     env = new Environment(envHome, envConfig);
400     }
401
402     private void createEnvAndDbs(Map JavaDoc[] dataMaps,
403                  boolean writeMultiple,
404                  boolean transactional,
405                  boolean abortBefore,
406                  boolean abortAfter,
407                  boolean corruptLog,
408                  Set JavaDoc lsnsToCorrupt,
409                  boolean deleteData)
410     throws DatabaseException {
411
412     createEnv(true, transactional);
413     Transaction txn = null;
414     if (transactional) {
415         txn = env.beginTransaction(null, null);
416     }
417
418     openDbs(true, transactional, duplicatesAllowed, txn);
419
420     if (transactional) {
421         txn.commit();
422     }
423
424     long lastCorruptedFile = -1;
425     for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
426         Map JavaDoc dataMap = new HashMap JavaDoc();
427         dataMaps[dbCnt] = dataMap;
428         Database db = dbs[dbCnt];
429
430         for (int i = 0; i < N_KEYS; i++) {
431         byte[] dataBytes = new byte[N_DATA_BYTES];
432         DatabaseEntry key = new DatabaseEntry();
433         DatabaseEntry data = new DatabaseEntry(dataBytes);
434         IntegerBinding.intToEntry(i, key);
435         TestUtils.generateRandomAlphaBytes(dataBytes);
436
437         boolean corruptedThisEntry = false;
438
439         if (transactional) {
440             txn = env.beginTransaction(null, null);
441         }
442
443         if (transactional &&
444             abortBefore) {
445             assertEquals(OperationStatus.SUCCESS,
446                  db.put(txn, key, data));
447             txn.abort();
448             txn = env.beginTransaction(null, null);
449         }
450
451         assertEquals(OperationStatus.SUCCESS,
452                  db.put(txn, key, data));
453         if (corruptLog) {
454             long currentLsn = getLastLsn();
455             long fileNumber = DbLsn.getFileNumber(currentLsn);
456             long fileOffset = DbLsn.getFileOffset(currentLsn);
457             if (fileOffset > (LOG_SIZE >> 1) &&
458             /* We're writing in the second half of the file. */
459             fileNumber > lastCorruptedFile) {
460             /* Corrupt this file. */
461             lsnsToCorrupt.add(new Long JavaDoc(currentLsn));
462             lastCorruptedFile = fileNumber;
463             corruptedThisEntry = true;
464             }
465         }
466
467         if (writeMultiple) {
468             assertEquals(OperationStatus.SUCCESS,
469                  db.delete(txn, key));
470             assertEquals(OperationStatus.SUCCESS,
471                  db.put(txn, key, data));
472         }
473
474         if (deleteData) {
475             assertEquals(OperationStatus.SUCCESS,
476                  db.delete(txn, key));
477             /* overload this for deleted data. */
478             corruptedThisEntry = true;
479         }
480
481         if (!corruptedThisEntry) {
482             dataMap.put(new Integer JavaDoc(i), new String JavaDoc(dataBytes));
483         }
484
485         if (transactional) {
486             txn.commit();
487         }
488
489         if (transactional &&
490             abortAfter) {
491             txn = env.beginTransaction(null, null);
492             assertEquals(OperationStatus.SUCCESS,
493                  db.put(txn, key, data));
494             txn.abort();
495         }
496         }
497     }
498     }
499
500     private void openDbs(boolean create,
501              boolean transactional,
502              boolean duplicatesAllowed,
503              Transaction txn)
504     throws DatabaseException {
505
506     for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
507         String JavaDoc databaseName = "simpleDb" + dbCnt;
508         DatabaseConfig dbConfig = new DatabaseConfig();
509         dbConfig.setAllowCreate(create);
510         dbConfig.setSortedDuplicates(duplicatesAllowed);
511         dbConfig.setTransactional(transactional);
512         if (dbs[dbCnt] != null) {
513         throw new DatabaseException("database already open");
514         }
515         dbs[dbCnt] = env.openDatabase(txn, databaseName, dbConfig);
516     }
517     }
518
519     private void dumpDbs(boolean printable, boolean aggressive)
520     throws DatabaseException {
521
522     try {
523         DbScavenger scavenger =
524                 new DbScavenger(env, null, envHomeName, printable, aggressive,
525                                 false /* verbose */);
526         scavenger.dump();
527     } catch (IOException JavaDoc IOE) {
528         throw new DatabaseException(IOE);
529     }
530     }
531
532     private void loadDbs()
533     throws DatabaseException {
534
535     try {
536         String JavaDoc dbNameBase = "simpleDb";
537         for (int i = 0; i < N_DBS; i++) {
538         DbLoad loader = new DbLoad();
539         File JavaDoc file = new File JavaDoc(envHomeName, dbNameBase + i + ".dump");
540         FileInputStream JavaDoc is = new FileInputStream JavaDoc(file);
541         BufferedReader JavaDoc reader =
542             new BufferedReader JavaDoc(new InputStreamReader JavaDoc(is));
543         loader.setEnv(env);
544         loader.setInputReader(reader);
545         loader.setNoOverwrite(false);
546         loader.setDbName(dbNameBase + i);
547         loader.load();
548         is.close();
549         }
550         } catch (IOException JavaDoc IOE) {
551         throw new DatabaseException(IOE);
552     }
553     }
554
555     private void verifyDbs(Map JavaDoc[] dataMaps)
556     throws DatabaseException {
557
558     for (int i = 0; i < N_DBS; i++) {
559         Map JavaDoc dataMap = dataMaps[i];
560         Cursor cursor = dbs[i].openCursor(null, null);
561         DatabaseEntry key = new DatabaseEntry();
562         DatabaseEntry data = new DatabaseEntry();
563         while (cursor.getNext(key, data, null) ==
564            OperationStatus.SUCCESS) {
565         Integer JavaDoc keyInt =
566             new Integer JavaDoc(IntegerBinding.entryToInt(key));
567         String JavaDoc databaseString = new String JavaDoc(data.getData());
568         String JavaDoc originalString = (String JavaDoc) dataMap.get(keyInt);
569         if (originalString == null) {
570             fail("couldn't find " + keyInt);
571         } else if (databaseString.equals(originalString)) {
572             dataMap.remove(keyInt);
573         } else {
574             fail(" Mismatch: key=" + keyInt +
575              " Expected: " + originalString +
576              " Found: " + databaseString);
577         }
578         }
579
580         if (dataMap.size() > 0) {
581         fail("entries still remain");
582         }
583
584         cursor.close();
585     }
586     }
587
588     private static DumpFileFilter dumpFileFilter = new DumpFileFilter();
589
590     static class DumpFileFilter implements FilenameFilter JavaDoc {
591
592     /**
593      * Accept files of this format:
594      * *.dump
595      */

596     public boolean accept(File JavaDoc dir, String JavaDoc name) {
597         StringTokenizer JavaDoc tokenizer = new StringTokenizer JavaDoc(name, ".");
598         /* There should be two parts. */
599         if (tokenizer.countTokens() == 2) {
600         String JavaDoc fileName = tokenizer.nextToken();
601         String JavaDoc fileSuffix = tokenizer.nextToken();
602
603         /* Check the length and the suffix. */
604         if (fileSuffix.equals("dump")) {
605             return true;
606         }
607         }
608
609         return false;
610     }
611     }
612
613     private long getLastLsn()
614     throws DatabaseException {
615
616     return DbInternal.envGetEnvironmentImpl(env).
617         getFileManager().getLastUsedLsn();
618     }
619
620     private void corruptFiles(Set JavaDoc lsnsToCorrupt)
621     throws DatabaseException {
622
623     Iterator JavaDoc iter = lsnsToCorrupt.iterator();
624     while (iter.hasNext()) {
625         long lsn = ((Long JavaDoc) iter.next()).longValue();
626         corruptFile(DbLsn.getFileNumber(lsn),
627             DbLsn.getFileOffset(lsn));
628     }
629     }
630
631     private void corruptFile(long fileNumber, long fileOffset)
632     throws DatabaseException {
633
634     String JavaDoc fileName = DbInternal.envGetEnvironmentImpl(env).
635         getFileManager().getFullFileName(fileNumber,
636                          FileManager.JE_SUFFIX);
637     /*
638     System.out.println("corrupting 1 byte at " +
639                DbLsn.makeLsn(fileNumber, fileOffset));
640     */

641     try {
642         RandomAccessFile JavaDoc raf = new RandomAccessFile JavaDoc(fileName, "rw");
643         raf.seek(fileOffset);
644         int current = raf.read();
645         raf.seek(fileOffset);
646         raf.write(current + 1);
647         raf.close();
648     } catch (IOException JavaDoc IOE) {
649         throw new DatabaseException(IOE);
650     }
651     }
652 }
653
Popular Tags