StatsFileReader


1   /*-
2    * See the file LICENSE for redistribution information.
3    *
4    * Copyright (c) 2002,2006 Oracle.  All rights reserved.
5    *
6    * $Id: StatsFileReader.java,v 1.14 2006/10/30 21:14:20 bostic Exp $
7    */
8   
9   package com.sleepycat.je.log;
10  
11  import java.io.IOException  ;
12  import java.nio.ByteBuffer  ;
13  import java.text.NumberFormat  ;
14  import java.util.ArrayList  ;
15  import java.util.Comparator  ;
16  import java.util.Iterator  ;
17  import java.util.Map  ;
18  import java.util.TreeMap  ;
19  
20  import com.sleepycat.je.DatabaseException;
21  import com.sleepycat.je.config.EnvironmentParams;
22  import com.sleepycat.je.dbi.EnvironmentImpl;
23  import com.sleepycat.je.utilint.DbLsn;
24  
25  /**
26   * The StatsFileReader generates stats about the log entries read, such as the
27   * count of each type of entry, the number of bytes, minimum and maximum sized
28   * log entry.
29   */
30  public class StatsFileReader extends DumpFileReader {
31  
32      /* Keyed by LogEntryType, data is EntryInfo. */
33      private Map   entryInfoMap;
34      private long totalLogBytes;
35      private long totalCount;
36  
37      /* Keep stats on log composition in terms of ckpt intervals. */
38      private ArrayList   ckptList;
39      private CheckpointCounter ckptCounter;
40      private long firstLsnRead;
41  
42      /**
43       * Create this reader to start at a given LSN.
44       */
45      public StatsFileReader(EnvironmentImpl env,
46                 int readBufferSize, 
47                 long startLsn,
48                 long finishLsn,
49                 String   entryTypes,
50                 String   txnIds,
51                 boolean verbose)
52      throws IOException  , DatabaseException {
53  
54          super(env, readBufferSize, startLsn, finishLsn,
55                entryTypes, txnIds, verbose);
56          entryInfoMap = new TreeMap  (new LogEntryTypeComparator());
57          totalLogBytes = 0;
58          totalCount = 0;
59  
60          ckptCounter = new CheckpointCounter();
61          ckptList = new ArrayList  ();
62          if (verbose) {
63              ckptList.add(ckptCounter);
64          }
65      }
66  
67      /**
68       * This reader collects stats about the log entry.
69       */
70      protected boolean processEntry(ByteBuffer   entryBuffer)
71          throws DatabaseException {
72  
73          /* 
74           * Record various stats based on the entry header, then move the buffer
75           * forward to skip ahead.
76           */
77          LogEntryType lastEntryType =
78              LogEntryType.findType(currentEntryTypeNum,
79                                    currentEntryTypeVersion);
80          entryBuffer.position(entryBuffer.position() + currentEntrySize);
81  
82          /* 
83           * Get the info object for it, if this is the first time it's seen,
84           * create an info object and insert it.
85           */
86          EntryInfo info = (EntryInfo) entryInfoMap.get(lastEntryType);
87          if (info == null) {
88              info = new EntryInfo();
89              entryInfoMap.put(lastEntryType, info);
90          }
91          
92          /* Update counts. */
93          info.count++;
94          totalCount++;
95          if (LogEntryType.isProvisional(currentEntryTypeVersion)) {
96              info.provisionalCount++;
97          }
98          int size = currentEntrySize + LogManager.HEADER_BYTES;
99          info.totalBytes += size;
100         totalLogBytes += size;
101         if ((info.minBytes==0) || (info.minBytes > size)) {
102             info.minBytes = size;
103         }
104         if (info.maxBytes < size) {
105             info.maxBytes = size;
106         }
107 
108         if (verbose) {
109             if (firstLsnRead == DbLsn.NULL_LSN) {
110                 firstLsnRead = getLastLsn();
111             } 
112 
113             if (currentEntryTypeNum ==
114                 LogEntryType.LOG_CKPT_END.getTypeNum()) {
115                 /* start counting a new interval */
116                 ckptCounter.endCkptLsn = getLastLsn();
117                 ckptCounter = new CheckpointCounter();
118                 ckptList.add(ckptCounter);
119             } else {
120                 ckptCounter.increment(this, currentEntryTypeNum);
121             }
122         }
123         
124         return true;
125     }
126 
127     public void summarize() {
128         System.out.println("Log statistics:");
129         Iterator   iter = entryInfoMap.entrySet().iterator();
130 
131         NumberFormat   form = NumberFormat.getIntegerInstance();
132         NumberFormat   percentForm = NumberFormat.getInstance();
133         percentForm.setMaximumFractionDigits(1);
134         System.out.println(pad("type") +
135                            pad("total") + 
136                            pad("provisional") + 
137                            pad("total") + 
138                            pad("min") + 
139                            pad("max") + 
140                            pad("avg") + 
141                            pad("entries"));
142 
143         System.out.println(pad("") +
144                            pad("count") + 
145                            pad("count") + 
146                            pad("bytes") + 
147                            pad("bytes") + 
148                            pad("bytes") + 
149                            pad("bytes") + 
150                            pad("as % of log"));
151 
152         long realTotalBytes = 0;
153 
154         while (iter.hasNext()) {
155             Map.Entry   m = (Map.Entry  ) iter.next();
156             EntryInfo info = (EntryInfo) m.getValue();
157             StringBuffer   sb = new StringBuffer  ();
158             LogEntryType entryType =(LogEntryType) m.getKey();
159             sb.append(pad(entryType.toString()));
160             sb.append(pad(form.format(info.count)));
161             sb.append(pad(form.format(info.provisionalCount)));
162             sb.append(pad(form.format(info.totalBytes)));
163             sb.append(pad(form.format(info.minBytes)));
164             sb.append(pad(form.format(info.maxBytes)));
165             sb.append(pad(form.format((long)(info.totalBytes/info.count))));
166             double entryPercent =
167                 ((double)(info.totalBytes *100)/totalLogBytes);
168             sb.append(pad(percentForm.format(entryPercent)));
169             System.out.println(sb.toString());
170 
171             /* Calculate key/data size for transactional LN */
172             if (entryType == LogEntryType.LOG_LN_TRANSACTIONAL) {
173                 /* 
174            LN_TX entry overhead:
175                    8 bytes node id
176                    1 byte boolean (whether data exists or is null)
177                    4 bytes data size
178                    4 bytes key size
179                    4 bytes database id
180                    8 bytes abort LSN
181                    1 byte abortKnownDeleted
182                    8 bytes txn id
183                    8 bytes lastlogged LSN (backpointer for txn)
184                 */
185                 int overhead = LogManager.HEADER_BYTES + 46;
186                 realTotalBytes += info.totalBytes-(info.count*overhead);
187             }
188 
189             /* Calculate key/data size for non-transactional LN */
190             if (entryType == LogEntryType.LOG_LN) {
191                 /* 
192            LN_TX entry overhead:
193                    8 bytes node id
194                    1 byte boolean (whether data exists or is null)
195                    4 bytes data size
196                    4 bytes key size
197                    4 bytes database id
198                 */
199                 int overhead = LogManager.HEADER_BYTES + 21;
200                 realTotalBytes += info.totalBytes-(info.count*overhead);
201             }
202         }
203 
204         /* Print special line for key/data */
205         StringBuffer   sb = new StringBuffer  ();
206         sb.append(pad("key/data"));
207         sb.append(pad(""));
208         sb.append(pad(""));
209         sb.append(pad(form.format(realTotalBytes)));
210         sb.append(pad(""));
211         sb.append(pad(""));
212         sb.append(pad(""));
213         String   realSize = "(" + 
214             percentForm.format((double)(realTotalBytes*100)/
215                                totalLogBytes) +
216             ")";
217         sb.append(pad(realSize));
218         System.out.println(sb.toString());
219 
220         System.out.println("\nTotal bytes in portion of log read: " +
221                            form.format(totalLogBytes));
222         System.out.println("Total number of entries: " +
223                            form.format(totalCount));
224 
225         if (verbose) {
226             summarizeCheckpointInfo();
227         }
228     }
229     
230     private String   pad(String   result) {
231         int spaces = 15 - result.length();
232         StringBuffer   sb = new StringBuffer  ();
233         for (int i = 0; i < spaces; i++) {
234             sb.append(" ");
235         }
236         sb.append(result);
237         return sb.toString();
238     }
239 
240     private void summarizeCheckpointInfo() {
241         System.out.println("\nPer checkpoint interval info:");
242 
243         /*
244          * Print out checkpoint interval info. 
245          * If the log looks like this:
246          * 
247          * start of log
248          * ckpt1 start
249          * ckpt1 end
250          * ckpt2 start
251          * ckpt2 end
252          * end of log
253          *
254          * There are 3 ckpt intervals
255          * start of log->ckpt1 end
256          * ckpt1 end -> ckpt2 end
257          * ckpt2 end -> end of log
258          */
259         System.out.println(pad("lnTxn") + 
260                            pad("ln") + 
261                            pad("mapLNTxn") + 
262                            pad("mapLN") +
263                            pad("end-end") +    // ckpt n-1 end -> ckpt n end
264                            pad("end-start") +  // ckpt n-1 end -> ckpt n start
265                            pad("start-end") +  // ckpt n start -> ckpt n end
266                            pad("maxLNReplay") +
267                            pad("ckptEnd"));
268 
269         long logFileMax;
270         try {
271             logFileMax = env.getConfigManager().getLong(
272                                    EnvironmentParams.LOG_FILE_MAX);
273         } catch (DatabaseException e) {
274             e.printStackTrace();
275             return;
276         }
277 
278         Iterator   iter = ckptList.iterator();
279         CheckpointCounter prevCounter = null;
280         NumberFormat   form = NumberFormat.getInstance();
281         while (iter.hasNext()) {
282             CheckpointCounter c = (CheckpointCounter)iter.next();
283             StringBuffer   sb = new StringBuffer  ();
284 
285             /* Entry type counts. */
286             int maxTxnLNs = c.preStartLNTxnCount + c.postStartLNTxnCount;
287             sb.append(pad(form.format(maxTxnLNs)));
288             int maxLNs = c.preStartLNCount + c.postStartLNCount;
289             sb.append(pad(form.format(maxLNs)));
290             sb.append(pad(form.format(c.preStartMapLNTxnCount +
291                                       c.postStartMapLNTxnCount)));
292             sb.append(pad(form.format(c.preStartMapLNCount +
293                                       c.postStartMapLNCount)));
294 
295             /* Checkpoint interval distance. */
296             long end = (c.endCkptLsn == DbLsn.NULL_LSN) ? getLastLsn() :
297                 c.endCkptLsn;
298             long endToEndDistance = 0;
299 
300             FileManager fileManager = env.getFileManager();
301             if (prevCounter == null) {
302                 endToEndDistance =
303                     DbLsn.getWithCleaningDistance(end,
304                           fileManager,
305                           firstLsnRead,
306                           logFileMax);
307             } else {
308                 endToEndDistance =
309                     DbLsn.getWithCleaningDistance(end,
310                           fileManager,
311                           prevCounter.endCkptLsn,
312                           logFileMax);
313             }
314             sb.append(pad(form.format(endToEndDistance)));
315 
316             /* 
317              * Interval between last checkpoint end and
318              * this checkpoint start. 
319              */
320             long start = (c.startCkptLsn == DbLsn.NULL_LSN) ? getLastLsn() :
321                 c.startCkptLsn;
322             long endToStartDistance = 0;
323 
324             if (prevCounter == null) {
325                 endToStartDistance =
326                     DbLsn.getWithCleaningDistance(start,
327                           fileManager,
328                                                   firstLsnRead,
329                                                   logFileMax);
330             } else {
331                 endToStartDistance =
332                     DbLsn.getWithCleaningDistance(start,
333                           fileManager,
334                                                   prevCounter.endCkptLsn,
335                                                   logFileMax);
336             }
337             sb.append(pad(form.format(endToStartDistance)));
338 
339             /* 
340              * Interval between ckpt start and ckpt end.
341              */
342             long startToEndDistance = 0;
343             if ((c.startCkptLsn != DbLsn.NULL_LSN)  &&
344                 (c.endCkptLsn != DbLsn.NULL_LSN)) {
345                 startToEndDistance =
346                     DbLsn.getWithCleaningDistance(c.endCkptLsn,
347                           fileManager,
348                           c.startCkptLsn,
349                           logFileMax);
350             }
351             sb.append(pad(form.format(startToEndDistance)));
352 
353             /* 
354              * The maximum number of LNs to replay includes the portion of LNs
355              * from checkpoint start to checkpoint end of the previous
356              * interval.
357              */
358             int maxReplay = maxLNs + maxTxnLNs;
359             if (prevCounter != null) {
360                 maxReplay += prevCounter.postStartLNTxnCount;
361                 maxReplay += prevCounter.postStartLNCount;
362             }
363             sb.append(pad(form.format(maxReplay)));
364 
365             if (c.endCkptLsn == DbLsn.NULL_LSN) {
366                 sb.append("   ").append(DbLsn.getNoFormatString(getLastLsn()));
367             } else {
368                 sb.append("   ").append(DbLsn.getNoFormatString(c.endCkptLsn));
369             }
370             
371             System.out.println(sb.toString());
372             prevCounter = c;
373         }
374     }
375         
376     static class EntryInfo {
377         public int count;
378         public int provisionalCount;
379         public long totalBytes;
380         public int minBytes;
381         public int maxBytes;
382 
383         EntryInfo() {
384             count = 0;
385             provisionalCount = 0;
386             totalBytes = 0;
387             minBytes = 0;
388             maxBytes = 0;
389         }
390     }
391         
392     static class LogEntryTypeComparator implements Comparator   {
393     public int compare(Object   o1, Object   o2) {
394         if (o1 == null) {
395         return -1;
396         }
397 
398         if (o2 == null) {
399         return 1;
400         }
401 
402         if (o1 instanceof LogEntryType &&
403         o2 instanceof LogEntryType) {
404         Byte   t1 = new Byte  (((LogEntryType) o1).getTypeNum());
405         Byte   t2 = new Byte  (((LogEntryType) o2).getTypeNum());
406         return t1.compareTo(t2);
407         } else {
408         throw new IllegalArgumentException  
409             ("non LogEntryType passed to LogEntryType.compare");
410         }
411     }
412     }
413 
414     /*
415      * Accumulate the count of items from checkpoint end->checkpoint end.
416      */
417     static class CheckpointCounter {
418         public long startCkptLsn = DbLsn.NULL_LSN;
419         public long endCkptLsn = DbLsn.NULL_LSN;
420         public int preStartLNTxnCount;
421         public int preStartLNCount;
422         public int preStartMapLNTxnCount;
423         public int preStartMapLNCount;
424         public int postStartLNTxnCount;
425         public int postStartLNCount;
426         public int postStartMapLNTxnCount;
427         public int postStartMapLNCount;
428 
429         public void increment(FileReader reader,  byte currentEntryTypeNum) {
430             if (currentEntryTypeNum == 
431                 LogEntryType.LOG_CKPT_START.getTypeNum()) {
432                 startCkptLsn = reader.getLastLsn();
433             } else if (currentEntryTypeNum == 
434                        LogEntryType.LOG_LN_TRANSACTIONAL.getTypeNum()) {
435                 if (startCkptLsn == DbLsn.NULL_LSN) {
436                     preStartLNTxnCount++;
437                 } else {
438                     postStartLNTxnCount++;
439                 }
440             } else if (currentEntryTypeNum == 
441                        LogEntryType.LOG_LN.getTypeNum()) {
442                 if (startCkptLsn == DbLsn.NULL_LSN) {
443                     preStartLNCount++;
444                 } else {
445                     postStartLNCount++;
446                 }
447             } else if (currentEntryTypeNum == 
448                        LogEntryType.LOG_MAPLN.getTypeNum()) {
449                 if (startCkptLsn == DbLsn.NULL_LSN) {
450                     preStartMapLNCount++;
451                 } else {
452                     postStartMapLNCount++;
453                 }
454             } else if (currentEntryTypeNum == 
455                        LogEntryType.LOG_MAPLN_TRANSACTIONAL.getTypeNum()) {
456                 if (startCkptLsn == DbLsn.NULL_LSN) {
457                     preStartMapLNTxnCount++;
458                 } else {
459                     postStartMapLNTxnCount++;
460                 }
461             }
462         }
463     }
464 }
465
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags