LNFileReaderTest


1   /*-
2    * See the file LICENSE for redistribution information.
3    *
4    * Copyright (c) 2002,2006 Oracle.  All rights reserved.
5    *
6    * $Id: LNFileReaderTest.java,v 1.85 2006/11/03 03:08:05 mark Exp $
7    */
8   
9   package com.sleepycat.je.log;
10  
11  import java.io.File  ;
12  import java.io.IOException  ;
13  import java.util.ArrayList  ;
14  import java.util.Arrays  ;
15  import java.util.List  ;
16  
17  import junit.framework.TestCase;
18  
19  import com.sleepycat.je.Database;
20  import com.sleepycat.je.DatabaseConfig;
21  import com.sleepycat.je.DatabaseException;
22  import com.sleepycat.je.DbInternal;
23  import com.sleepycat.je.Environment;
24  import com.sleepycat.je.EnvironmentConfig;
25  import com.sleepycat.je.TransactionConfig;
26  import com.sleepycat.je.config.EnvironmentParams;
27  import com.sleepycat.je.dbi.DbConfigManager;
28  import com.sleepycat.je.dbi.EnvironmentImpl;
29  import com.sleepycat.je.tree.LN;
30  import com.sleepycat.je.tree.MapLN;
31  import com.sleepycat.je.txn.LockType;
32  import com.sleepycat.je.txn.Txn;
33  import com.sleepycat.je.util.TestUtils;
34  import com.sleepycat.je.utilint.DbLsn;
35  import com.sleepycat.je.utilint.Tracer;
36  
37  /**
38   * Test the LNFileReader
39   */
40  public class LNFileReaderTest extends TestCase {
41      static private final boolean DEBUG = false;
42  
43      private File   envHome;
44      private Environment env;
45      private EnvironmentImpl envImpl;
46      private Database db;
47      private List   checkList;
48  
49      public LNFileReaderTest() {
50          super();
51          envHome = new File  (System.getProperty(TestUtils.DEST_DIR));
52      }
53  
54      public void setUp()
55          throws IOException  , DatabaseException {
56  
57          /*
58           * Note that we use the official Environment class to make the
59           * environment, so that everything is set up, but we then go a backdoor
60           * route to get to the underlying EnvironmentImpl class so that we
61           * don't require that the Environment.getDbEnvironment method be
62           * unnecessarily public.
63           */
64          TestUtils.removeLogFiles("Setup", envHome, false);
65          EnvironmentConfig envConfig = TestUtils.initEnvConfig();
66      DbInternal.disableParameterValidation(envConfig);
67      envConfig.setConfigParam(EnvironmentParams.NODE_MAX.getName(), "6");
68          envConfig.setConfigParam
69          (EnvironmentParams.LOG_FILE_MAX.getName(), "1024");
70          envConfig.setAllowCreate(true);
71      envConfig.setTransactional(true);
72          env = new Environment(envHome, envConfig);
73  
74          envImpl = DbInternal.envGetEnvironmentImpl(env);
75      }
76      
77      public void tearDown()
78          throws IOException  , DatabaseException {
79  
80          envImpl = null;
81          env.close();
82          TestUtils.removeFiles("TearDown", envHome, FileManager.JE_SUFFIX);
83      }
84  
85      /**
86       * Test no log file
87       */
88      public void testNoFile()
89          throws IOException  , DatabaseException {
90  
91          /* Make a log file with a valid header, but no data. */
92          LNFileReader reader =
93              new LNFileReader(envImpl,
94                               1000,             // read buffer size
95                               DbLsn.NULL_LSN,   // start lsn
96                               true,             // redo
97                               DbLsn.NULL_LSN,   // end of file lsn
98                               DbLsn.NULL_LSN,   // finish lsn
99                               null);            // single file
100         reader.addTargetType(LogEntryType.LOG_LN_TRANSACTIONAL);
101         reader.addTargetType(LogEntryType.LOG_DEL_DUPLN_TRANSACTIONAL);
102         assertFalse("Empty file should not have entries",
103                     reader.readNextEntry());
104     }
105 
106     /**
107      * Run with an empty file.
108      */
109     public void testEmpty()
110         throws IOException  , DatabaseException {
111 
112         /* Make a log file with a valid header, but no data. */
113         FileManager fileManager = envImpl.getFileManager();
114         FileManagerTestUtils.createLogFile(fileManager, envImpl, 1000);
115         fileManager.clear();
116 
117         LNFileReader reader =
118             new LNFileReader(envImpl, 
119                              1000,             // read buffer size
120                              DbLsn.NULL_LSN,   // start lsn
121                              true,             // redo
122                              DbLsn.NULL_LSN,   // end of file lsn
123                              DbLsn.NULL_LSN,   // finish lsn
124                              null);            // single file
125         reader.addTargetType(LogEntryType.LOG_LN_TRANSACTIONAL);
126         reader.addTargetType(LogEntryType.LOG_DEL_DUPLN_TRANSACTIONAL);
127         assertFalse("Empty file should not have entries",
128                     reader.readNextEntry());
129     }
130 
131     /**
132      * Run with defaults, read whole log for redo, going forwards.
133      */
134     public void testBasicRedo()
135         throws Throwable   {
136 
137         try {
138             DbConfigManager cm =  envImpl.getConfigManager();
139             doTest(50,
140                    cm.getInt(EnvironmentParams.LOG_ITERATOR_READ_SIZE),
141                    0,
142                    false,
143                    true);
144         } catch (Throwable   t) {
145             t.printStackTrace();
146             throw t;
147         }
148     }
149 
150     /**
151      * Run with defaults, read whole log for undo, going backwards.
152      */
153     public void testBasicUndo()
154         throws Throwable   {
155 
156         try {
157             DbConfigManager cm =  envImpl.getConfigManager();
158             doTest(50, 
159                    cm.getInt(EnvironmentParams.LOG_ITERATOR_READ_SIZE),
160                    0,
161                    false,
162                    false);
163         } catch (Throwable   t) {
164             t.printStackTrace();
165             throw t;
166         }
167     }
168 
169     /**
170      * Run with very small read buffer for redo, and track LNs.
171      */
172     public void testSmallBuffersRedo()
173         throws IOException  , DatabaseException {
174 
175         doTest(50, 10, 0, true, true);
176     }
177 
178     /**
179      * Run with very small read buffer for undo and track LNs.
180      */
181     public void testSmallBuffersUndo()
182         throws IOException  , DatabaseException {
183 
184         doTest(50, 10, 0, true, false);
185     }
186 
187 
188     /**
189      * Run with medium buffers for redo.
190      */
191     public void testMedBuffersRedo()
192         throws IOException  , DatabaseException {
193 
194         doTest(50, 100, 0, false, true);
195     }
196 
197     /**
198      * Run with medium buffers for undo.
199      */
200     public void testMedBuffersUndo()
201         throws IOException  , DatabaseException {
202 
203         doTest(50, 100, 0, false, false);
204     }
205 
206     /**
207      * Start in the middle of the file for redo.
208      */
209     public void testMiddleStartRedo()
210         throws IOException  , DatabaseException {
211 
212         doTest(50, 100, 20, true, true);
213     }
214 
215     /**
216      * Start in the middle of the file for undo.
217      */
218     public void testMiddleStartUndo()
219         throws IOException  , DatabaseException {
220 
221         doTest(50, 100, 20, true, false);
222     }
223 
224     /**
225      * Create a log file, create the reader, read the log file
226      * @param numIters each iteration makes 3 log entries (debug record, ln
227      *           and mapLN
228      * @param bufferSize to pass to reader
229      * @param checkIndex where in the test data to start
230      * @param trackLNs true if we're tracking LNS, false if we're tracking 
231      *           mapLNs
232      */
233     private void doTest(int numIters,
234             int bufferSize,
235             int checkIndex,
236                         boolean trackLNs,
237             boolean redo)
238         throws IOException  , DatabaseException {
239 
240         checkList = new ArrayList  ();
241 
242         /* Fill up a fake log file. */
243         long endOfFileLsn = createLogFile(numIters, trackLNs, redo);
244 
245         if (DEBUG) {
246             System.out.println("eofLsn = " + endOfFileLsn);
247         }
248 
249         /* Decide where to start. */
250         long startLsn = DbLsn.NULL_LSN;
251         long finishLsn = DbLsn.NULL_LSN;
252         if (redo) {
253             startLsn = ((CheckInfo) checkList.get(checkIndex)).lsn;
254         } else {
255             /* Going backwards. Start at last check entry. */
256             int lastEntryIdx = checkList.size() - 1;
257             startLsn = ((CheckInfo) checkList.get(lastEntryIdx)).lsn;
258             finishLsn = ((CheckInfo) checkList.get(checkIndex)).lsn;
259         }
260 
261         LNFileReader reader =
262         new LNFileReader(envImpl, bufferSize, startLsn, redo, endOfFileLsn,
263                  finishLsn, null);
264         if (trackLNs) {
265             reader.addTargetType(LogEntryType.LOG_LN_TRANSACTIONAL);
266             reader.addTargetType(LogEntryType.LOG_DEL_DUPLN_TRANSACTIONAL);
267         } else {
268             reader.addTargetType(LogEntryType.LOG_MAPLN_TRANSACTIONAL);
269         }
270 
271         if (!redo) {
272             reader.addTargetType(LogEntryType.LOG_TXN_COMMIT);
273         }
274         
275         /* read. */
276         checkLogFile(reader, checkIndex, redo);
277     }
278 
279     /**
280      * Write a logfile of entries, put the entries that we expect to
281      * read into a list for later verification.
282      * @return end of file LSN.
283      */
284     private long createLogFile(int numIters, boolean trackLNs, boolean redo)
285         throws IOException  , DatabaseException {
286 
287         /*
288          * Create a log file full of LNs, DeletedDupLNs, MapLNs and Debug
289          * Records
290          */
291         DatabaseConfig dbConfig = new DatabaseConfig();
292         dbConfig.setAllowCreate(true);
293         db = env.openDatabase(null, "foo", dbConfig);
294         LogManager logManager = envImpl.getLogManager();
295 
296         long lsn;
297         Txn userTxn = new Txn(envImpl, new TransactionConfig());
298         long txnId = userTxn.getId();
299 
300         for (int i = 0; i < numIters; i++) {
301             /* Add a debug record just to be filler. */
302             Tracer rec = new Tracer("Hello there, rec " + (i+1));
303             logManager.log(rec);
304 
305             /* Make a transactional LN, we expect it to be there. */
306             byte[] data = new byte[i+1];
307             Arrays.fill(data, (byte)(i+1));
308             LN ln = new LN(data);
309             byte[] key = new byte[i+1];
310             Arrays.fill(key, (byte)(i+10));
311             
312             /* 
313          * Log an LN. If we're tracking LNs add it to the verification
314          * list.
315          */
316             userTxn.lock
317                 (ln.getNodeId(), LockType.WRITE, false,
318                  DbInternal.dbGetDatabaseImpl(db));
319             lsn = ln.log(envImpl,
320                          DbInternal.dbGetDatabaseImpl(db).getId(),
321                          key,
322                          DbLsn.NULL_LSN,
323                          0,
324                          userTxn,
325                          false);
326 
327             if (trackLNs) {
328                 checkList.add(new CheckInfo(lsn, ln, key,
329                                             ln.getData(), txnId));
330             }
331 
332             /* Log a deleted duplicate LN. */
333             LN deleteLN = new LN(data);
334             byte[] dupKey = new byte[i+1];
335             Arrays.fill(dupKey, (byte)(i+2));
336 
337             userTxn.lock
338                 (deleteLN.getNodeId(), LockType.WRITE, false,
339                  DbInternal.dbGetDatabaseImpl(db));
340             lsn = deleteLN.delete(DbInternal.dbGetDatabaseImpl(db),
341                                   key, 
342                                   dupKey,
343                                   DbLsn.NULL_LSN,
344                                   userTxn);
345             if (trackLNs) {
346                 checkList.add(new CheckInfo(lsn, deleteLN,
347                                             dupKey, key, txnId));
348             }
349 
350             /* 
351          * Make a non-transactional LN. Shouldn't get picked up by reader.
352          */
353             LN nonTxnalLN = new LN(data);
354             nonTxnalLN.log(envImpl,
355                DbInternal.dbGetDatabaseImpl(db).getId(),
356                key, DbLsn.NULL_LSN, 0, null, false);
357 
358             /* Add a MapLN. */
359             MapLN mapLN = new MapLN(DbInternal.dbGetDatabaseImpl(db));
360             userTxn.lock
361                 (mapLN.getNodeId(), LockType.WRITE, false,
362                  DbInternal.dbGetDatabaseImpl(db));
363             lsn = mapLN.log(envImpl,
364                             DbInternal.dbGetDatabaseImpl(db).getId(),
365                             key, DbLsn.NULL_LSN, 0, userTxn, false);
366             if (!trackLNs) {
367                 checkList.add(new CheckInfo(lsn, mapLN, key,
368                                             mapLN.getData(),
369                                             txnId));
370             }
371         }
372 
373         long commitLsn = userTxn.commit(Txn.TXN_SYNC);
374 
375         /* We only expect checkpoint entries to be read in redo passes. */
376         if (!redo) {
377             checkList.add(new CheckInfo(commitLsn, null, null, null, txnId));
378         }
379 
380         /* Make a marker log entry to pose as the end of file. */
381         Tracer rec = new Tracer("Pretend this is off the file");
382         long lastLsn = logManager.log(rec);
383         db.close();
384         logManager.flush();
385         envImpl.getFileManager().clear();
386         return lastLsn;
387     }
388 
389 
390     private void checkLogFile(LNFileReader reader,
391                               int checkIndex,
392                               boolean redo)
393         throws IOException  , DatabaseException {
394 
395         LN lnFromLog;
396         byte[] keyFromLog;
397                 
398         /* Read all the LNs. */
399         int i;
400         if (redo) {
401             /* start where indicated. */
402             i = checkIndex;
403         } else {
404             /* start at the end. */
405             i = checkList.size() - 1;
406         }
407         while (reader.readNextEntry()) {
408             CheckInfo expected = (CheckInfo) checkList.get(i);
409 
410             /* Check LSN. */
411             assertEquals("LSN " + i + " should match",
412                          expected.lsn,
413                          reader.getLastLsn());
414 
415             if (reader.isLN()) {
416 
417                 /* Check the LN. */
418                 lnFromLog = reader.getLN();
419                 LN expectedLN = expected.ln;
420                 assertEquals("Should be the same type of object",
421                              expectedLN.getClass(),
422                              lnFromLog.getClass());
423 
424                 if (DEBUG) {
425                     if (!expectedLN.toString().equals(lnFromLog.toString())) {
426                         System.out.println("expected = " +
427                                            expectedLN.toString()+
428                                            "lnFromLog = " +
429                                            lnFromLog.toString());
430                     }
431                 }
432                 assertEquals("LN " + i + " should match",
433                              expectedLN.toString(),
434                              lnFromLog.toString());
435 
436                 /* Check the key. */
437                 keyFromLog = reader.getKey();
438                 byte[] expectedKey = expected.key;
439                 if (DEBUG) {
440                     if (!Arrays.equals(expectedKey, keyFromLog)) {
441                         System.out.println("expectedKey=" + expectedKey +
442                                            " logKey=" + keyFromLog);
443                     }
444                 }
445                     
446                 assertTrue("Key " + i + " should match",
447                            Arrays.equals(expectedKey, keyFromLog));
448                 
449                 /* Check the dup key. */
450                 byte[] dupKeyFromLog = reader.getDupTreeKey();
451                 byte[] expectedDupKey = expected.dupKey;
452                 assertTrue(Arrays.equals(expectedDupKey, dupKeyFromLog));
453 
454                 assertEquals(expected.txnId,
455                              reader.getTxnId().longValue());
456                 
457             } else {
458                 /* Should be a txn commit record. */
459                 assertEquals(expected.txnId,
460                              reader.getTxnCommitId());
461             }
462 
463             if (redo) {
464                 i++;
465             } else {
466                 i--;
467             }
468         }
469         int expectedCount = checkList.size() - checkIndex;
470         assertEquals(expectedCount, reader.getNumRead());
471     }
472 
473     private class CheckInfo {
474         long lsn;
475         LN ln;
476         byte[] key;
477         byte[] dupKey;
478         long txnId;
479         
480         CheckInfo(long lsn, LN ln, byte[] key, byte[] dupKey, long txnId) {
481             this.lsn = lsn;
482             this.ln = ln;
483             this.key = key;
484             this.dupKey = dupKey;
485             this.txnId = txnId;
486         }
487     }
488 }
489
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags