|
1
8
9 package com.sleepycat.je.test;
10
11 import java.io.File  ;
12 import java.util.HashSet ;
13 import java.util.Random ;
14 import java.util.Set ;
15 import java.util.logging.Level ;
16
17 import junit.framework.TestCase;
18
19 import com.sleepycat.bind.tuple.IntegerBinding;
20 import com.sleepycat.je.CheckpointConfig;
21 import com.sleepycat.je.Cursor;
22 import com.sleepycat.je.Database;
23 import com.sleepycat.je.DatabaseConfig;
24 import com.sleepycat.je.DatabaseEntry;
25 import com.sleepycat.je.DatabaseException;
26 import com.sleepycat.je.DbInternal;
27 import com.sleepycat.je.Environment;
28 import com.sleepycat.je.EnvironmentConfig;
29 import com.sleepycat.je.EnvironmentStats;
30 import com.sleepycat.je.LockMode;
31 import com.sleepycat.je.OperationStatus;
32 import com.sleepycat.je.PreloadConfig;
33 import com.sleepycat.je.PreloadStats;
34 import com.sleepycat.je.StatsConfig;
35 import com.sleepycat.je.Transaction;
36 import com.sleepycat.je.config.EnvironmentParams;
37 import com.sleepycat.je.dbi.MemoryBudget;
38 import com.sleepycat.je.log.FileManager;
39 import com.sleepycat.je.util.TestUtils;
40 import com.sleepycat.je.utilint.Tracer;
41
42 public class DeferredWriteTest extends TestCase {
43 private static boolean DEBUG = false;
44 private static String DBNAME = "foo";
45
46 private static final CheckpointConfig CHECKPOINT_FORCE_CONFIG =
47 new CheckpointConfig();
48
49 static {
50 CHECKPOINT_FORCE_CONFIG.setForce(true);
51 }
52
53 private static final StatsConfig STATS_CLEAR_CONFIG = new StatsConfig();
54
55 static {
56 STATS_CLEAR_CONFIG.setClear(true);
57 }
58
59 private File envHome;
60 private Environment env;
61
62 public DeferredWriteTest()
63 throws Exception {
64
65 envHome = new File (System.getProperty(TestUtils.DEST_DIR));
66 }
67
68 public void setUp()
69 throws Exception {
70 TestUtils.removeFiles("Setup", envHome, FileManager.JE_SUFFIX);
71 TestUtils.removeFiles("Setup", envHome, FileManager.DEL_SUFFIX);
72 }
73
74 public void tearDown()
75 throws Exception {
76
77 if (env != null) {
78 try {
79 env.close();
80 } catch (Exception e) {
81 System.err.println("TearDown: " + e);
82 }
83 }
84 env = null;
85 TestUtils.removeFiles("TearDown", envHome, FileManager.JE_SUFFIX);
86 }
87
88 private EnvironmentConfig getEnvConfig(boolean transactional) {
89 EnvironmentConfig envConfig = TestUtils.initEnvConfig();
90 envConfig.setTransactional(transactional);
91 envConfig.setAllowCreate(true);
92 envConfig.setConfigParam(EnvironmentParams.NODE_MAX.getName(), "4");
93 if (DEBUG) {
94 envConfig.setConfigParam("java.util.logging.ConsoleHandler.on",
95 "true");
96 envConfig.setConfigParam("java.util.logging.level.cleaner",
97 "SEVERE");
98 }
99
100 return envConfig;
101 }
102
103 private Database createDb(boolean deferredWrite)
104 throws DatabaseException {
105
106 DatabaseConfig dbConfig = new DatabaseConfig();
107 dbConfig.setAllowCreate(true);
108 dbConfig.setDeferredWrite(deferredWrite);
109
110 return env.openDatabase(null, DBNAME, dbConfig);
111 }
112
113 public void testEmptyDatabaseSR14744()
114 throws Throwable {
115
116 Database db = null;
117 try {
118 EnvironmentConfig envConfig = getEnvConfig(true);
119 env = new Environment(envHome, envConfig);
120 db = createDb(true);
121 db.sync();
122 } finally {
123 if (db != null) {
124 db.close();
125 }
126
127 env.sync();
128 env.close();
129 env = null;
130 }
131 }
132
133
136 public void testCloseOpen()
137 throws Throwable {
138
139 HashSet expectedSet =
140 doCloseOpen(true,
141 1,
142 new HashSet ());
143 expectedSet =
144 doCloseOpen(false,
145 100,
146 expectedSet);
147 expectedSet =
148 doCloseOpen(true,
149 200,
150 expectedSet);
151
152 }
153
154
158 private HashSet doCloseOpen(boolean useDeferredWrite,
159 int startingValue,
160 HashSet initialSet)
161 throws Throwable {
162
163 EnvironmentConfig envConfig = getEnvConfig(true);
164 env = new Environment(envHome, envConfig);
165 Database db = createDb(useDeferredWrite);
166
167
168 HashSet expectedBatch1 = new HashSet ();
169 expectedBatch1.addAll(initialSet);
170 HashSet expectedBatch2 = new HashSet ();
171 HashSet finalExpectedSet = null;
172
173 int batch1Size = 40;
174 int batch2Size = 50;
175
176 try {
177
178
183 insert(db, null, startingValue, startingValue + batch1Size,
184 expectedBatch1, false);
185 checkExactContentMatch(db, expectedBatch1);
186 if (useDeferredWrite) {
187 db.sync();
188 }
189
190
191 insert(db, null,
192 startingValue + batch1Size,
193 startingValue + batch2Size,
194 expectedBatch2, false);
195 expectedBatch2.addAll(expectedBatch1);
196 checkExactContentMatch(db, expectedBatch2);
197
198
199 db.close();
200 db = createDb(useDeferredWrite);
201 checkExactContentMatch(db, expectedBatch2);
202
203
208
209 db.close();
210 db = null;
211
212 env.sync();
213 env.close();
214 env = null;
215 env = new Environment(envHome, envConfig);
216
217 db = createDb(useDeferredWrite);
218
219 if (useDeferredWrite) {
220 finalExpectedSet = expectedBatch1;
221 } else {
222 finalExpectedSet = expectedBatch2;
223 }
224
225 checkExactContentMatch(db, finalExpectedSet);
226 db.close();
227 db = null;
228 } finally {
229 if (db != null) {
230 db.close();
231 }
232
233 env.sync();
234 env.close();
235 env = null;
236 }
237 return finalExpectedSet;
238 }
239
240 public void testRecoverNoSync()
241 throws Throwable {
242
243 EnvironmentConfig envConfig = getEnvConfig(true);
244 doRecover(envConfig,
245 30,
246 false,
247 false);
248 }
249
250 public void testRecoverSync()
251 throws Throwable {
252
253 EnvironmentConfig envConfig = getEnvConfig(true);
254 doRecover(envConfig,
255 30,
256 true,
257 false);
258 }
259
260 public void testRecoverNoSyncEvict()
261 throws Throwable {
262
263 EnvironmentConfig envConfig = getEnvConfig(true);
264 envConfig.setCacheSize(MemoryBudget.MIN_MAX_MEMORY_SIZE);
265 doRecover(envConfig,
266 3000,
267 false,
268 true);
269 }
270
271 public void testRecoverSyncEvict()
272 throws Throwable {
273
274 EnvironmentConfig envConfig = getEnvConfig(true);
275 envConfig.setCacheSize(MemoryBudget.MIN_MAX_MEMORY_SIZE);
276 doRecover(envConfig,
277 3000,
278 true,
279 true);
280 }
281
282 public void doRecover(EnvironmentConfig envConfig,
283 int numRecords,
284 boolean syncBeforeRecovery,
285 boolean expectEviction)
286 throws DatabaseException {
287
288 env = new Environment(envHome, envConfig);
289 Database db = createDb(true);
290 HashSet expected = new HashSet ();
291
292 try {
293
294 EnvironmentStats stats = env.getStats(STATS_CLEAR_CONFIG);
295 insert(db, null, 1, numRecords, expected, true);
296 checkForEvictionActivity(expectEviction,
297 expectEviction);
298 checkExactContentMatch(db, expected);
299 checkForEvictionActivity(expectEviction,
300 expectEviction);
301
302
303 env.checkpoint(CHECKPOINT_FORCE_CONFIG);
304 DatabaseConfig saveConfig = db.getConfig();
305 if (syncBeforeRecovery) {
306 db.sync();
307 }
308
309
312 db.close();
313 env.checkpoint(CHECKPOINT_FORCE_CONFIG);
314 DbInternal.envGetEnvironmentImpl(env).close(false);
315 env = null;
316
317
318 env = new Environment(envHome, envConfig);
319 db = env.openDatabase(null, DBNAME, saveConfig);
320
321
322 HashSet useExpected = null;
323 if (syncBeforeRecovery) {
324 useExpected = expected;
325 } else {
326 useExpected = new HashSet ();
327 }
328
329 checkExactContentMatch(db, useExpected);
330
331 } finally {
332 db.close();
333 }
334 }
335
336 public void testPreloadNoSync()
337 throws DatabaseException {
338
339 doPreload(false);
340 }
341
342 public void testPreloadSync()
343 throws DatabaseException {
344
345 doPreload(true);
346 }
347
348 private void doPreload(boolean syncBeforeRecovery)
349 throws DatabaseException {
350
351 EnvironmentConfig envConfig = getEnvConfig(false);
352 envConfig.setCacheSize(MemoryBudget.MIN_MAX_MEMORY_SIZE);
353 env = new Environment(envHome, envConfig);
354 Database db = createDb(true);
355 HashSet expected = new HashSet ();
356
357 int numRecords = 3000;
358
359 try {
360
361 EnvironmentStats stats = env.getStats(STATS_CLEAR_CONFIG);
362 insert(db, null, 1, numRecords, expected, true);
363 checkForEvictionActivity(true,
364 true);
365
366
370 envConfig.setCacheSize(0);
371 env.setMutableConfig(envConfig);
372 if (DEBUG) {
373 System.out.println("after mutable " +
374 env.getConfig().getCacheSize());
375 }
376
377 PreloadConfig pConfig = new PreloadConfig();
378 pConfig.setLoadLNs(true);
379 PreloadStats pStats = db.preload(pConfig);
380
381 if (DEBUG) {
382 System.out.println("first preload " + pStats);
383 }
384 assertTrue(pStats.getNBINsLoaded() > 50);
385 assertTrue(pStats.getNINsLoaded() > 50);
386 assertTrue(pStats.getNLNsLoaded() > 50);
387
388 checkExactContentMatch(db, expected);
389
390 DatabaseConfig saveConfig = db.getConfig();
391 if (syncBeforeRecovery) {
392 db.sync();
393 }
394
395
396 db.close();
397 DbInternal.envGetEnvironmentImpl(env).close(false);
398 env = null;
399
400
401 env = new Environment(envHome, envConfig);
402 db = env.openDatabase(null, DBNAME, saveConfig);
403 pStats = db.preload(pConfig);
404 if (DEBUG) {
405 System.out.println("second preload " + pStats);
406 }
407
408
409 HashSet useExpected = null;
410 if (syncBeforeRecovery) {
411 useExpected = expected;
412 assertTrue(pStats.getNBINsLoaded() > 50);
413 assertTrue(pStats.getNINsLoaded() > 50);
414 assertTrue(pStats.getNLNsLoaded() > 50);
415 } else {
416 useExpected = new HashSet ();
417 assertEquals(0, pStats.getNBINsLoaded());
418 assertEquals(0, pStats.getNINsLoaded());
419 assertEquals(0, pStats.getNLNsLoaded());
420 }
421
422 checkExactContentMatch(db, useExpected);
423
424 } finally {
425 db.close();
426 }
427 }
428
429 private void checkForEvictionActivity(boolean expectEviction,
430 boolean expectCacheMiss)
431 throws DatabaseException {
432
433 EnvironmentStats stats = env.getStats(STATS_CLEAR_CONFIG);
434 if (DEBUG) {
435 System.out.println("EvictPasses=" + stats.getNEvictPasses());
436 System.out.println("Selected=" + stats.getNNodesSelected());
437 System.out.println("Stripped=" + stats.getNBINsStripped());
438 System.out.println("Evicted=" +
439 stats.getNNodesExplicitlyEvicted());
440 System.out.println("CacheMiss=" +
441 stats.getNCacheMiss());
442 }
443
444 if (expectEviction) {
445
446 assertTrue(stats.getNNodesSelected() > 50);
447 assertTrue(stats.getNBINsStripped() > 50);
448 assertTrue(stats.getNNodesExplicitlyEvicted() > 50);
449 }
450
451 if (expectCacheMiss) {
452 assertTrue(stats.getNCacheMiss()>50);
453 }
454 }
455
456 public void testBadConfigurations()
457 throws Throwable {
458
459 env = new Environment(envHome, getEnvConfig(true));
460
461 DatabaseConfig dbConfigDeferred = new DatabaseConfig();
462 dbConfigDeferred.setAllowCreate(true);
463 dbConfigDeferred.setDeferredWrite(true);
464
465 DatabaseConfig dbConfigNoDeferred = new DatabaseConfig();
466 dbConfigNoDeferred.setAllowCreate(true);
467
468
469 try {
470 dbConfigDeferred.setTransactional(true);
471 Database db = env.openDatabase(null, "foo", dbConfigDeferred);
472 fail("No support yet for txnal, deferred-write databases");
473 } catch (DatabaseException expected) {
474 if (DEBUG) {
475 System.out.println("expected=" + expected);
476 }
477 }
478 dbConfigDeferred.setTransactional(false);
479
480
484 Database db1 = env.openDatabase(null, "foo", dbConfigDeferred);
485 try {
486 Database db2 = env.openDatabase(null, "foo", dbConfigNoDeferred);
487 fail("Database already opened with deferred write");
488 } catch (DatabaseException expected) {
489 if (DEBUG) {
490 System.out.println("expected=" + expected);
491 }
492 }
493 db1.close();
494
495
499 db1 = env.openDatabase(null, "foo", dbConfigNoDeferred);
500 try {
501 Database db2 = env.openDatabase(null, "foo", dbConfigDeferred);
502 fail("Database already opened with out deferred write");
503 } catch (DatabaseException expected) {
504 if (DEBUG) {
505 System.out.println("expected=" + expected);
506 }
507 }
508 db1.close();
509
510
511 Database db = null;
512 try {
513 db = env.openDatabase(null, "foo", dbConfigNoDeferred);
514 db.sync();
515 fail("Sync not permitted");
516 } catch (DatabaseException expected) {
517 if (DEBUG) {
518 System.out.println("expected=" + expected);
519 }
520 } finally {
521 db.close();
522 }
523 }
524
525 public void testCleaning5000()
526 throws Throwable {
527
528 doCleaning("90", "5000");
529 }
530
531 public void testCleaning2000()
532 throws Throwable {
533
534 doCleaning("90", "3000");
535 }
536
537 private void doCleaning(String minUtilization, String logFileSize)
538 throws DatabaseException {
539
540
545 long cacheSize = MemoryBudget.MIN_MAX_MEMORY_SIZE +
546 (MemoryBudget.MIN_MAX_MEMORY_SIZE / 2);
547 EnvironmentConfig envConfig = getEnvConfig(true);
548 DbInternal.disableParameterValidation(envConfig);
549 envConfig.setCacheSize(cacheSize);
550 envConfig.setConfigParam("je.cleaner.minUtilization",
551 minUtilization);
552 envConfig.setConfigParam("je.log.fileMax", logFileSize);
553 envConfig.setConfigParam("je.cleaner.expunge", "false");
554
555 envConfig.setConfigParam("je.env.runCleaner", "false");
556 env = new Environment(envHome, envConfig);
557 Database db = createDb(true);
558
559
560 HashSet expectedBatch1 = new HashSet ();
561 HashSet expectedBatch2 = new HashSet ();
562 HashSet finalExpectedSet = null;
563
564 int batch1Size = 100;
565 int batch2Size = 100;
566
567 try {
568
569
574 int startingValue = 1;
575 insert(db,
576 null,
577 startingValue,
578 startingValue + batch1Size,
579 expectedBatch1,
580 false);
581 checkExactContentMatch(db, expectedBatch1);
582 db.sync();
583
584
585 insertAndUpdate(db,
586 null,
587 startingValue + batch1Size,
588 startingValue + batch2Size,
589 expectedBatch2,
590 false);
591 expectedBatch2.addAll(expectedBatch1);
592 checkExactContentMatch(db, expectedBatch2);
593 env.checkpoint(CHECKPOINT_FORCE_CONFIG);
594 Tracer.trace(Level.SEVERE,
595 DbInternal.envGetEnvironmentImpl(env),
596 "before clean");
597 batchClean();
598
599 Tracer.trace(Level.SEVERE,
600 DbInternal.envGetEnvironmentImpl(env),
601 "after clean");
602
603 checkExactContentMatch(db, expectedBatch2);
604
605
610 for (int i = 0; i < 4; i++) {
611 db.close();
612 db = null;
613
614 env.close();
615 env = new Environment(envHome, envConfig);
616
617 db = createDb(true);
618 checkContents(db,
619 expectedBatch2,
620 false);
621
622 batchClean();
623 checkContents(db,
624 expectedBatch2,
625 false);
626 }
627
628 db.close();
629 db = null;
630 } finally {
631 if (db != null) {
632 db.close();
633 }
634
635 env.close();
636 env = null;
637 }
638 }
639
640
644 private void insert(Database db,
645 Transaction txn,
646 int start,
647 int end,
648 Set expected,
649 boolean useRandom)
650 throws DatabaseException{
651
652 DatabaseEntry entry = new DatabaseEntry();
653 Random rand = new Random ();
654 for (int i = start; i < end; i++) {
655 int value = useRandom ? rand.nextInt() : i;
656
657 IntegerBinding.intToEntry(value, entry);
658 assertEquals(OperationStatus.SUCCESS,
659 db.put(txn, entry, entry));
660 expected.add(new Integer (value));
661 }
662 }
663
664
669 private void insertAndUpdate(Database db,
670 Transaction txn,
671 int start,
672 int end,
673 Set expected,
674 boolean useRandom)
675 throws DatabaseException{
676
677 DatabaseEntry key = new DatabaseEntry();
678 DatabaseEntry data = new DatabaseEntry();
679 Random rand = new Random ();
680 for (int i = start; i < end; i++) {
681 int value = useRandom ? rand.nextInt() : i;
682
683 IntegerBinding.intToEntry(value, key);
684 IntegerBinding.intToEntry(value - 1, data);
685 OperationStatus status = db.putNoOverwrite(txn, key, data);
686 if (status == OperationStatus.SUCCESS) {
687
688 IntegerBinding.intToEntry(value, data);
689 db.put(txn, key, data);
690 expected.add(new Integer (value));
691 }
692 }
693 }
694
695
698 private void checkExactContentMatch(Database db, HashSet expected)
699 throws DatabaseException{
700
701 checkContents(db, expected, true);
702 }
703
704
710 private void checkContents(Database db,
711 HashSet expected,
712 boolean exactMatch)
713 throws DatabaseException{
714
715 Cursor c = db.openCursor(null, null);
716 DatabaseEntry key = new DatabaseEntry();
717 DatabaseEntry data = new DatabaseEntry();
718
719 Set useExpected = (Set ) expected.clone();
720
721 if (DEBUG) {
722 System.err.println("Start checking");
723 }
724
725 try {
726 while (c.getNext(key, data, LockMode.DEFAULT) ==
727 OperationStatus.SUCCESS) {
728 int value = IntegerBinding.entryToInt(key);
729
730 if (DEBUG) {
731 System.err.println("checkDatabase: found " + value);
732 }
733
734 assertTrue(value + " not in useExpected set. Expected size="
735 + useExpected.size(),
736 useExpected.remove(new Integer (value)));
737 assertEquals(value, IntegerBinding.entryToInt(data));
738 }
739
740 if (exactMatch) {
741 assertEquals(0, useExpected.size());
742 } else {
743 if (DEBUG) {
744 System.out.println(useExpected.size() +
745 " is leftover in expected set");
746 }
747 }
748 } finally {
749 c.close();
750 }
751 }
752
753 private void batchClean()
754 throws DatabaseException {
755
756 int cleaned = 0;
757 int cleanedThisRound = 0;
758 do {
759 cleanedThisRound = env.cleanLog();
760 cleaned += cleanedThisRound;
761 } while (cleanedThisRound > 0);
762
763 if (DEBUG) {
764 System.out.println("numCleaned = " + cleaned);
765 }
766
767 assertTrue("cleaned must be > 0, was only " + cleaned +
768 " but may vary on machine to machine", cleaned > 0);
769
770 if (cleaned > 0) {
771 CheckpointConfig force = new CheckpointConfig();
772 force.setForce(true);
773 env.checkpoint(force);
774 }
775 }
776 }
777 |
Popular Tags |
Java API By Example, From Geeks To Geeks.