|
1
8
9 package com.sleepycat.je.util;
10
11 import java.io.BufferedReader  ;
12 import java.io.File ;
13 import java.io.FileInputStream ;
14 import java.io.FilenameFilter ;
15 import java.io.IOException ;
16 import java.io.InputStreamReader ;
17 import java.io.RandomAccessFile ;
18 import java.util.HashMap ;
19 import java.util.HashSet ;
20 import java.util.Iterator ;
21 import java.util.Map ;
22 import java.util.Set ;
23 import java.util.StringTokenizer ;
24
25 import junit.framework.TestCase;
26
27 import com.sleepycat.bind.tuple.IntegerBinding;
28 import com.sleepycat.je.Cursor;
29 import com.sleepycat.je.Database;
30 import com.sleepycat.je.DatabaseConfig;
31 import com.sleepycat.je.DatabaseEntry;
32 import com.sleepycat.je.DatabaseException;
33 import com.sleepycat.je.DatabaseNotFoundException;
34 import com.sleepycat.je.DbInternal;
35 import com.sleepycat.je.Environment;
36 import com.sleepycat.je.EnvironmentConfig;
37 import com.sleepycat.je.OperationStatus;
38 import com.sleepycat.je.Transaction;
39 import com.sleepycat.je.config.EnvironmentParams;
40 import com.sleepycat.je.log.FileManager;
41 import com.sleepycat.je.utilint.DbLsn;
42 import com.sleepycat.je.utilint.DbScavenger;
43
44 public class DbScavengerTest extends TestCase {
45
46 private static final int TRANSACTIONAL = 1 << 0;
47 private static final int WRITE_MULTIPLE = 1 << 1;
48 private static final int PRINTABLE = 1 << 2;
49 private static final int ABORT_BEFORE = 1 << 3;
50 private static final int ABORT_AFTER = 1 << 4;
51 private static final int CORRUPT_LOG = 1 << 5;
52 private static final int DELETE_DATA = 1 << 6;
53 private static final int AGGRESSIVE = 1 << 7;
54
55 private static final int N_DBS = 3;
56 private static final int N_KEYS = 100;
57 private static final int N_DATA_BYTES = 100;
58 private static final int LOG_SIZE = 10000;
59
60 private String envHomeName;
61 private File envHome;
62
63 private Environment env;
64
65 private Database[] dbs = new Database[N_DBS];
66
67 private boolean duplicatesAllowed = true;
68
69 public DbScavengerTest() {
70 envHomeName = System.getProperty(TestUtils.DEST_DIR);
71 envHome = new File (envHomeName);
72 }
73
74 public void setUp()
75 throws IOException {
76
77 TestUtils.removeLogFiles("Setup", envHome, false);
78 TestUtils.removeFiles("Setup", envHome, ".dump");
79 }
80
81 public void tearDown()
82 throws IOException {
83
84 if (env != null) {
85 try {
86 env.close();
87 } catch (Exception e) {
88 System.out.println("TearDown: " + e);
89 }
90 env = null;
91 }
92 TestUtils.removeLogFiles("TearDown", envHome, false);
93 TestUtils.removeFiles("Teardown", envHome, ".dump");
94 }
95
96 public void testScavenger1()
97 throws Throwable {
98
99 try {
100 doScavengerTest(PRINTABLE | TRANSACTIONAL |
101 ABORT_BEFORE | ABORT_AFTER);
102 } catch (Throwable T) {
103 System.out.println("caught " + T);
104 T.printStackTrace();
105 }
106 }
107
108 public void testScavenger2()
109 throws Throwable {
110
111 try {
112 doScavengerTest(PRINTABLE | TRANSACTIONAL | ABORT_BEFORE);
113 } catch (Throwable T) {
114 System.out.println("caught " + T);
115 T.printStackTrace();
116 }
117 }
118
119 public void testScavenger3()
120 throws Throwable {
121
122 try {
123 doScavengerTest(PRINTABLE | TRANSACTIONAL | ABORT_AFTER);
124 } catch (Throwable T) {
125 System.out.println("caught " + T);
126 T.printStackTrace();
127 }
128 }
129
130 public void testScavenger4()
131 throws Throwable {
132
133 try {
134 doScavengerTest(PRINTABLE | TRANSACTIONAL);
135 } catch (Throwable T) {
136 System.out.println("caught " + T);
137 T.printStackTrace();
138 }
139 }
140
141 public void testScavenger5()
142 throws Throwable {
143
144 try {
145 doScavengerTest(PRINTABLE | WRITE_MULTIPLE | TRANSACTIONAL);
146 } catch (Throwable T) {
147 System.out.println("caught " + T);
148 T.printStackTrace();
149 }
150 }
151
152 public void testScavenger6()
153 throws Throwable {
154
155 try {
156 doScavengerTest(PRINTABLE);
157 } catch (Throwable T) {
158 System.out.println("caught " + T);
159 T.printStackTrace();
160 throw T;
161 }
162 }
163
164 public void testScavenger7()
165 throws Throwable {
166
167 try {
168 doScavengerTest(TRANSACTIONAL | ABORT_BEFORE | ABORT_AFTER);
169 } catch (Throwable T) {
170 System.out.println("caught " + T);
171 T.printStackTrace();
172 }
173 }
174
175 public void testScavenger8()
176 throws Throwable {
177
178 try {
179 doScavengerTest(TRANSACTIONAL | ABORT_BEFORE);
180 } catch (Throwable T) {
181 System.out.println("caught " + T);
182 T.printStackTrace();
183 }
184 }
185
186 public void testScavenger9()
187 throws Throwable {
188
189 try {
190 doScavengerTest(TRANSACTIONAL);
191 } catch (Throwable T) {
192 System.out.println("caught " + T);
193 T.printStackTrace();
194 }
195 }
196
197 public void testScavenger10()
198 throws Throwable {
199
200 try {
201 doScavengerTest(TRANSACTIONAL | ABORT_AFTER);
202 } catch (Throwable T) {
203 System.out.println("caught " + T);
204 T.printStackTrace();
205 }
206 }
207
208 public void testScavenger11()
209 throws Throwable {
210
211 try {
212 doScavengerTest(0);
213 } catch (Throwable T) {
214 System.out.println("caught " + T);
215 T.printStackTrace();
216 }
217 }
218
219 public void testScavenger12()
220 throws Throwable {
221
222 try {
223 doScavengerTest(CORRUPT_LOG);
224 } catch (Throwable T) {
225 System.out.println("caught " + T);
226 T.printStackTrace();
227 }
228 }
229
230 public void testScavenger13()
231 throws Throwable {
232
233 try {
234 doScavengerTest(DELETE_DATA);
235 } catch (Throwable T) {
236 System.out.println("caught " + T);
237 T.printStackTrace();
238 }
239 }
240
241 public void testScavenger14()
242 throws Throwable {
243
244 try {
245 doScavengerTest(AGGRESSIVE);
246 } catch (Throwable T) {
247 System.out.println("caught " + T);
248 T.printStackTrace();
249 }
250 }
251
252 public void testScavengerAbortedDbLevelOperations()
253 throws Throwable {
254
255 try {
256 createEnv(true, true);
257 boolean doAbort = true;
258 byte[] dataBytes = new byte[N_DATA_BYTES];
259 DatabaseEntry key = new DatabaseEntry();
260 DatabaseEntry data = new DatabaseEntry(dataBytes);
261 IntegerBinding.intToEntry(1, key);
262 TestUtils.generateRandomAlphaBytes(dataBytes);
263 for (int i = 0; i < 2; i++) {
264 Transaction txn = env.beginTransaction(null, null);
265 for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
266 String databaseName = null;
267 if (doAbort) {
268 databaseName = "abortedDb" + dbCnt;
269 } else {
270 databaseName = "simpleDb" + dbCnt;
271 }
272 DatabaseConfig dbConfig = new DatabaseConfig();
273 dbConfig.setAllowCreate(true);
274 dbConfig.setSortedDuplicates(duplicatesAllowed);
275 dbConfig.setTransactional(true);
276 if (dbs[dbCnt] != null) {
277 throw new DatabaseException("database already open");
278 }
279 Database db =
280 env.openDatabase(txn, databaseName, dbConfig);
281 dbs[dbCnt] = db;
282 db.put(txn, key, data);
283 }
284 if (doAbort) {
285 txn.abort();
286 dbs = new Database[N_DBS];
287 } else {
288 txn.commit();
289 }
290 doAbort = !doAbort;
291 }
292
293 closeEnv();
294 createEnv(false, false);
295 openDbs(false, false, duplicatesAllowed, null);
296 dumpDbs(false, false);
297
298
299 closeEnv();
300 TestUtils.removeLogFiles("doScavengerTest", envHome, false);
301
302
303 createEnv(true, true);
304 loadDbs();
305
306
307 for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
308 String databaseName = "abortedDb" + dbCnt;
309 DatabaseConfig dbConfig = new DatabaseConfig();
310 dbConfig.setAllowCreate(false);
311 try {
312 env.openDatabase(null, databaseName, dbConfig);
313 fail("expected DatabaseNotFoundException");
314 } catch (DatabaseNotFoundException DNFE) {
315
316 }
317 }
318 closeEnv();
319
320 } catch (Throwable T) {
321 System.out.println("caught " + T);
322 T.printStackTrace();
323 }
324 }
325
326 private void doScavengerTest(int config)
327 throws DatabaseException, IOException {
328
329 boolean printable = (config & PRINTABLE) != 0;
330 boolean transactional = (config & TRANSACTIONAL) != 0;
331 boolean writeMultiple = (config & WRITE_MULTIPLE) != 0;
332 boolean abortBefore = (config & ABORT_BEFORE) != 0;
333 boolean abortAfter = (config & ABORT_AFTER) != 0;
334 boolean corruptLog = (config & CORRUPT_LOG) != 0;
335 boolean deleteData = (config & DELETE_DATA) != 0;
336 boolean aggressive = (config & AGGRESSIVE) != 0;
337
338 assert transactional ||
339 (!abortBefore && !abortAfter);
340
341 Map [] dataMaps = new Map [N_DBS];
342 Set lsnsToCorrupt = new HashSet ();
343
344 createEnvAndDbs(dataMaps,
345 writeMultiple,
346 transactional,
347 abortBefore,
348 abortAfter,
349 corruptLog,
350 lsnsToCorrupt,
351 deleteData);
352 closeEnv();
353 createEnv(false, false);
354 if (corruptLog) {
355 corruptFiles(lsnsToCorrupt);
356 }
357 openDbs(false, false, duplicatesAllowed, null);
358 dumpDbs(printable, aggressive);
359
360
361 closeEnv();
362 TestUtils.removeLogFiles("doScavengerTest", envHome, false);
363
364
365 createEnv(true, transactional);
366 loadDbs();
367
368
369 openDbs(false, false, duplicatesAllowed, null);
370 verifyDbs(dataMaps);
371 closeEnv();
372 }
373
374 private void closeEnv()
375 throws DatabaseException {
376
377 for (int i = 0; i < N_DBS; i++) {
378 if (dbs[i] != null) {
379 dbs[i].close();
380 dbs[i] = null;
381 }
382 }
383
384 env.close();
385 env = null;
386 }
387
388 private void createEnv(boolean create, boolean transactional)
389 throws DatabaseException {
390
391 EnvironmentConfig envConfig = TestUtils.initEnvConfig();
392 DbInternal.disableParameterValidation(envConfig);
393 envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER.getName(),
394 "false");
395 envConfig.setConfigParam(EnvironmentParams.LOG_FILE_MAX.getName(),
396 "" + LOG_SIZE);
397 envConfig.setTransactional(transactional);
398 envConfig.setAllowCreate(create);
399 env = new Environment(envHome, envConfig);
400 }
401
402 private void createEnvAndDbs(Map [] dataMaps,
403 boolean writeMultiple,
404 boolean transactional,
405 boolean abortBefore,
406 boolean abortAfter,
407 boolean corruptLog,
408 Set lsnsToCorrupt,
409 boolean deleteData)
410 throws DatabaseException {
411
412 createEnv(true, transactional);
413 Transaction txn = null;
414 if (transactional) {
415 txn = env.beginTransaction(null, null);
416 }
417
418 openDbs(true, transactional, duplicatesAllowed, txn);
419
420 if (transactional) {
421 txn.commit();
422 }
423
424 long lastCorruptedFile = -1;
425 for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
426 Map dataMap = new HashMap ();
427 dataMaps[dbCnt] = dataMap;
428 Database db = dbs[dbCnt];
429
430 for (int i = 0; i < N_KEYS; i++) {
431 byte[] dataBytes = new byte[N_DATA_BYTES];
432 DatabaseEntry key = new DatabaseEntry();
433 DatabaseEntry data = new DatabaseEntry(dataBytes);
434 IntegerBinding.intToEntry(i, key);
435 TestUtils.generateRandomAlphaBytes(dataBytes);
436
437 boolean corruptedThisEntry = false;
438
439 if (transactional) {
440 txn = env.beginTransaction(null, null);
441 }
442
443 if (transactional &&
444 abortBefore) {
445 assertEquals(OperationStatus.SUCCESS,
446 db.put(txn, key, data));
447 txn.abort();
448 txn = env.beginTransaction(null, null);
449 }
450
451 assertEquals(OperationStatus.SUCCESS,
452 db.put(txn, key, data));
453 if (corruptLog) {
454 long currentLsn = getLastLsn();
455 long fileNumber = DbLsn.getFileNumber(currentLsn);
456 long fileOffset = DbLsn.getFileOffset(currentLsn);
457 if (fileOffset > (LOG_SIZE >> 1) &&
458
459 fileNumber > lastCorruptedFile) {
460
461 lsnsToCorrupt.add(new Long (currentLsn));
462 lastCorruptedFile = fileNumber;
463 corruptedThisEntry = true;
464 }
465 }
466
467 if (writeMultiple) {
468 assertEquals(OperationStatus.SUCCESS,
469 db.delete(txn, key));
470 assertEquals(OperationStatus.SUCCESS,
471 db.put(txn, key, data));
472 }
473
474 if (deleteData) {
475 assertEquals(OperationStatus.SUCCESS,
476 db.delete(txn, key));
477
478 corruptedThisEntry = true;
479 }
480
481 if (!corruptedThisEntry) {
482 dataMap.put(new Integer (i), new String (dataBytes));
483 }
484
485 if (transactional) {
486 txn.commit();
487 }
488
489 if (transactional &&
490 abortAfter) {
491 txn = env.beginTransaction(null, null);
492 assertEquals(OperationStatus.SUCCESS,
493 db.put(txn, key, data));
494 txn.abort();
495 }
496 }
497 }
498 }
499
500 private void openDbs(boolean create,
501 boolean transactional,
502 boolean duplicatesAllowed,
503 Transaction txn)
504 throws DatabaseException {
505
506 for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
507 String databaseName = "simpleDb" + dbCnt;
508 DatabaseConfig dbConfig = new DatabaseConfig();
509 dbConfig.setAllowCreate(create);
510 dbConfig.setSortedDuplicates(duplicatesAllowed);
511 dbConfig.setTransactional(transactional);
512 if (dbs[dbCnt] != null) {
513 throw new DatabaseException("database already open");
514 }
515 dbs[dbCnt] = env.openDatabase(txn, databaseName, dbConfig);
516 }
517 }
518
519 private void dumpDbs(boolean printable, boolean aggressive)
520 throws DatabaseException {
521
522 try {
523 DbScavenger scavenger =
524 new DbScavenger(env, null, envHomeName, printable, aggressive,
525 false );
526 scavenger.dump();
527 } catch (IOException IOE) {
528 throw new DatabaseException(IOE);
529 }
530 }
531
532 private void loadDbs()
533 throws DatabaseException {
534
535 try {
536 String dbNameBase = "simpleDb";
537 for (int i = 0; i < N_DBS; i++) {
538 DbLoad loader = new DbLoad();
539 File file = new File (envHomeName, dbNameBase + i + ".dump");
540 FileInputStream is = new FileInputStream (file);
541 BufferedReader reader =
542 new BufferedReader (new InputStreamReader (is));
543 loader.setEnv(env);
544 loader.setInputReader(reader);
545 loader.setNoOverwrite(false);
546 loader.setDbName(dbNameBase + i);
547 loader.load();
548 is.close();
549 }
550 } catch (IOException IOE) {
551 throw new DatabaseException(IOE);
552 }
553 }
554
555 private void verifyDbs(Map [] dataMaps)
556 throws DatabaseException {
557
558 for (int i = 0; i < N_DBS; i++) {
559 Map dataMap = dataMaps[i];
560 Cursor cursor = dbs[i].openCursor(null, null);
561 DatabaseEntry key = new DatabaseEntry();
562 DatabaseEntry data = new DatabaseEntry();
563 while (cursor.getNext(key, data, null) ==
564 OperationStatus.SUCCESS) {
565 Integer keyInt =
566 new Integer (IntegerBinding.entryToInt(key));
567 String databaseString = new String (data.getData());
568 String originalString = (String ) dataMap.get(keyInt);
569 if (originalString == null) {
570 fail("couldn't find " + keyInt);
571 } else if (databaseString.equals(originalString)) {
572 dataMap.remove(keyInt);
573 } else {
574 fail(" Mismatch: key=" + keyInt +
575 " Expected: " + originalString +
576 " Found: " + databaseString);
577 }
578 }
579
580 if (dataMap.size() > 0) {
581 fail("entries still remain");
582 }
583
584 cursor.close();
585 }
586 }
587
588 private static DumpFileFilter dumpFileFilter = new DumpFileFilter();
589
590 static class DumpFileFilter implements FilenameFilter {
591
592
596 public boolean accept(File dir, String name) {
597 StringTokenizer tokenizer = new StringTokenizer (name, ".");
598
599 if (tokenizer.countTokens() == 2) {
600 String fileName = tokenizer.nextToken();
601 String fileSuffix = tokenizer.nextToken();
602
603
604 if (fileSuffix.equals("dump")) {
605 return true;
606 }
607 }
608
609 return false;
610 }
611 }
612
613 private long getLastLsn()
614 throws DatabaseException {
615
616 return DbInternal.envGetEnvironmentImpl(env).
617 getFileManager().getLastUsedLsn();
618 }
619
620 private void corruptFiles(Set lsnsToCorrupt)
621 throws DatabaseException {
622
623 Iterator iter = lsnsToCorrupt.iterator();
624 while (iter.hasNext()) {
625 long lsn = ((Long ) iter.next()).longValue();
626 corruptFile(DbLsn.getFileNumber(lsn),
627 DbLsn.getFileOffset(lsn));
628 }
629 }
630
631 private void corruptFile(long fileNumber, long fileOffset)
632 throws DatabaseException {
633
634 String fileName = DbInternal.envGetEnvironmentImpl(env).
635 getFileManager().getFullFileName(fileNumber,
636 FileManager.JE_SUFFIX);
637
641 try {
642 RandomAccessFile raf = new RandomAccessFile (fileName, "rw");
643 raf.seek(fileOffset);
644 int current = raf.read();
645 raf.seek(fileOffset);
646 raf.write(current + 1);
647 raf.close();
648 } catch (IOException IOE) {
649 throw new DatabaseException(IOE);
650 }
651 }
652 }
653 |
Popular Tags |
Java API By Example, From Geeks To Geeks.