|
1
8
9 package com.sleepycat.je.utilint;
10
11 import java.io.File  ;
12 import java.io.FileOutputStream ;
13 import java.io.IOException ;
14 import java.io.PrintStream ;
15 import java.util.Date ;
16 import java.util.HashMap ;
17 import java.util.Iterator ;
18 import java.util.Map ;
19
20 import com.sleepycat.je.DatabaseException;
21 import com.sleepycat.je.DbInternal;
22 import com.sleepycat.je.Environment;
23 import com.sleepycat.je.config.EnvironmentParams;
24 import com.sleepycat.je.dbi.DatabaseImpl;
25 import com.sleepycat.je.dbi.DbConfigManager;
26 import com.sleepycat.je.dbi.EnvironmentImpl;
27 import com.sleepycat.je.log.FileManager;
28 import com.sleepycat.je.log.LastFileReader;
29 import com.sleepycat.je.log.LogEntryType;
30 import com.sleepycat.je.log.ScavengerFileReader;
31 import com.sleepycat.je.log.entry.LNLogEntry;
32 import com.sleepycat.je.log.entry.LogEntry;
33 import com.sleepycat.je.tree.LN;
34 import com.sleepycat.je.tree.MapLN;
35 import com.sleepycat.je.tree.NameLN;
36 import com.sleepycat.je.util.DbDump;
37
38 public class DbScavenger extends DbDump {
39 private static final int FLUSH_INTERVAL = 100;
40 private int readBufferSize;
41 private EnvironmentImpl envImpl;
42
43
46 private BitMap committedTxnIdsSeen;
47
48
51 private BitMap nodeIdsSeen;
52
53
56 private Map dbIdToName;
57
58
61 private Map dbIdToDupSort;
62
63
66 private Map dbIdToOutputStream;
67
68 private boolean dumpCorruptedBounds = false;
69
70 private int flushCounter = 0;
71 private long lastTime;
72
73 public DbScavenger(Environment env,
74 PrintStream outputFile,
75 String outputDirectory,
76 boolean formatUsingPrintable,
77 boolean doAggressiveScavengerRun,
78 boolean verbose) {
79 super(env, null, outputFile, outputDirectory, formatUsingPrintable);
80
81 this.doAggressiveScavengerRun = doAggressiveScavengerRun;
82 this.dbIdToName = new HashMap ();
83 this.dbIdToDupSort = new HashMap ();
84 this.dbIdToOutputStream = new HashMap ();
85 this.verbose = verbose;
86 }
87
88
91 public void setDumpCorruptedBounds(boolean dumpCorruptedBounds) {
92 this.dumpCorruptedBounds = dumpCorruptedBounds;
93 }
94
95 public void dump()
96 throws IOException , DatabaseException {
97
98 openEnv(false);
99
100 envImpl = DbInternal.envGetEnvironmentImpl(env);
101 DbConfigManager cm = envImpl.getConfigManager();
102 try {
103 readBufferSize =
104 cm.getInt(EnvironmentParams.LOG_ITERATOR_READ_SIZE);
105 } catch (DatabaseException DBE) {
106 readBufferSize = 8192;
107 }
108
109
112 LastFileReader reader = new LastFileReader(envImpl, readBufferSize);
113 while (reader.readNextEntry()) {
114 }
115
116
117 long lastUsedLsn = reader.getLastValidLsn();
118 long nextAvailableLsn = reader.getEndOfLog();
119 envImpl.getFileManager().setLastPosition(nextAvailableLsn,
120 lastUsedLsn,
121 reader.getPrevOffset());
122
123 try {
124
125 if (verbose) {
126 System.out.println("Pass 1: " + new Date ());
127 }
128 scavengeDbTree(lastUsedLsn, nextAvailableLsn);
129
130
131 if (verbose) {
132 System.out.println("Pass 2: " + new Date ());
133 }
134 scavenge(lastUsedLsn, nextAvailableLsn);
135
136 if (verbose) {
137 System.out.println("End: " + new Date ());
138 }
139 } finally {
140 closeOutputStreams();
141 }
142 }
143
144
148 private void scavengeDbTree(long lastUsedLsn, long nextAvailableLsn)
149 throws IOException , DatabaseException {
150
151 committedTxnIdsSeen = new BitMap();
152 nodeIdsSeen = new BitMap();
153
154 final ScavengerFileReader scavengerReader =
155 new ScavengerFileReader(envImpl, readBufferSize, lastUsedLsn,
156 DbLsn.NULL_LSN, nextAvailableLsn) {
157 protected void processEntryCallback(LogEntry entry,
158 LogEntryType entryType)
159 throws DatabaseException {
160
161 processDbTreeEntry(entry, entryType);
162 }
163 };
164
165 scavengerReader.setTargetType(LogEntryType.LOG_MAPLN_TRANSACTIONAL);
166 scavengerReader.setTargetType(LogEntryType.LOG_MAPLN);
167 scavengerReader.setTargetType(LogEntryType.LOG_NAMELN_TRANSACTIONAL);
168 scavengerReader.setTargetType(LogEntryType.LOG_NAMELN);
169 scavengerReader.setTargetType(LogEntryType.LOG_TXN_COMMIT);
170 scavengerReader.setTargetType(LogEntryType.LOG_TXN_ABORT);
171 lastTime = System.currentTimeMillis();
172 long fileNum = -1;
173 while (scavengerReader.readNextEntry()) {
174 fileNum = reportProgress(fileNum,
175 scavengerReader.getLastLsn());
176 }
177 }
178
179 private long reportProgress(long fileNum, long lastLsn) {
180
181 long currentFile = DbLsn.getFileNumber(lastLsn);
182 if (verbose) {
183 if (currentFile != fileNum) {
184 long now = System.currentTimeMillis();
185 System.out.println("processing file " +
186 FileManager.getFileName(currentFile,
187 ".jdb ") +
188 (now-lastTime) + " ms");
189 lastTime = now;
190 }
191 }
192
193 return currentFile;
194 }
195
196
199 private boolean checkProcessEntry(LogEntry entry,
200 LogEntryType entryType,
201 boolean pass2) {
202 boolean isTransactional = entry.isTransactional();
203
204
211 if (isTransactional) {
212 long txnId = entry.getTransactionId();
213 if (entryType.equals(LogEntryType.LOG_TXN_COMMIT)) {
214 committedTxnIdsSeen.set(txnId);
215
216 return false;
217 }
218
219 if (entryType.equals(LogEntryType.LOG_TXN_ABORT)) {
220
221 return false;
222 }
223
224 if (!committedTxnIdsSeen.get(txnId)) {
225 return false;
226 }
227 }
228
229
232 if (entry instanceof LNLogEntry) {
233 LNLogEntry lnEntry = (LNLogEntry) entry;
234 LN ln = lnEntry.getLN();
235 long nodeId = ln.getNodeId();
236 boolean isDelDupLN =
237 entryType.equals(LogEntryType.
238 LOG_DEL_DUPLN_TRANSACTIONAL) ||
239 entryType.equals(LogEntryType.LOG_DEL_DUPLN);
240
241
245 if (pass2 && doAggressiveScavengerRun) {
246 return !isDelDupLN;
247 }
248 if (nodeIdsSeen.get(nodeId)) {
249 return false;
250 } else {
251 nodeIdsSeen.set(nodeId);
252 if (isDelDupLN) {
253
254
258 return false;
259 } else {
260 return true;
261 }
262 }
263 }
264
265 return false;
266 }
267
268
271 private void processDbTreeEntry(LogEntry entry, LogEntryType entryType)
272 throws DatabaseException {
273
274 boolean processThisEntry =
275 checkProcessEntry(entry, entryType, false);
276
277 if (processThisEntry &&
278 (entry instanceof LNLogEntry)) {
279 LNLogEntry lnEntry = (LNLogEntry) entry;
280 LN ln = lnEntry.getLN();
281 if (ln instanceof NameLN) {
282 String name = new String (lnEntry.getKey());
283 Integer dbId = new Integer (((NameLN) ln).getId().getId());
284 if (dbIdToName.containsKey(dbId) &&
285 !((String ) dbIdToName.get(dbId)).equals(name)) {
286 throw new DatabaseException
287 ("Already name mapped for dbId: " + dbId +
288 " changed from " + (String ) dbIdToName.get(dbId) +
289 " to " + name);
290 } else {
291 dbIdToName.put(dbId, name);
292 }
293 }
294
295 if (ln instanceof MapLN) {
296 DatabaseImpl db = ((MapLN) ln).getDatabase();
297 Integer dbId = new Integer (db.getId().getId());
298 Boolean dupSort = Boolean.valueOf(db.getSortedDuplicates());
299 if (dbIdToDupSort.containsKey(dbId)) {
300 throw new DatabaseException
301 ("Already saw dupSort entry for dbId: " + dbId);
302 } else {
303 dbIdToDupSort.put(dbId, dupSort);
304 }
305 }
306 }
307 }
308
309
312 private void scavenge(long lastUsedLsn, long nextAvailableLsn)
313 throws IOException , DatabaseException {
314
315 final ScavengerFileReader scavengerReader =
316 new ScavengerFileReader(envImpl, readBufferSize, lastUsedLsn,
317 DbLsn.NULL_LSN, nextAvailableLsn) {
318 protected void processEntryCallback(LogEntry entry,
319 LogEntryType entryType)
320 throws DatabaseException {
321
322 processRegularEntry(entry, entryType);
323 }
324 };
325
326
330 scavengerReader.setTargetType(LogEntryType.LOG_LN_TRANSACTIONAL);
331 scavengerReader.setTargetType(LogEntryType.LOG_LN);
332 scavengerReader.setTargetType
333 (LogEntryType.LOG_DEL_DUPLN_TRANSACTIONAL);
334 scavengerReader.setTargetType(LogEntryType.LOG_DEL_DUPLN);
335 scavengerReader.setDumpCorruptedBounds(dumpCorruptedBounds);
336
337 long progressFileNum = -1;
338 while (scavengerReader.readNextEntry()) {
339 progressFileNum = reportProgress(progressFileNum,
340 scavengerReader.getLastLsn());
341 }
342 }
343
344
347 private void processRegularEntry(LogEntry entry, LogEntryType entryType)
348 throws DatabaseException {
349
350 boolean processThisEntry =
351 checkProcessEntry(entry, entryType, true);
352
353 if (processThisEntry) {
354 LNLogEntry lnEntry = (LNLogEntry) entry;
355 Integer dbId = new Integer (lnEntry.getDbId().getId());
356 PrintStream out = getOutputStream(dbId);
357
358 LN ln = lnEntry.getLN();
359 byte[] keyData = lnEntry.getKey();
360 byte[] data = ln.getData();
361 if (data != null) {
362 dumpOne(out, keyData, formatUsingPrintable);
363 dumpOne(out, data, formatUsingPrintable);
364 if ((++flushCounter % FLUSH_INTERVAL) == 0) {
365 out.flush();
366 flushCounter = 0;
367 }
368 }
369 }
370 }
371
372
376 private PrintStream getOutputStream(Integer dbId)
377 throws DatabaseException {
378
379 try {
380 PrintStream ret = (PrintStream ) dbIdToOutputStream.get(dbId);
381 if (ret != null) {
382 return ret;
383 }
384 String name = (String ) dbIdToName.get(dbId);
385 if (name == null) {
386 name = "db" + dbId;
387 }
388 File file = new File (outputDirectory, name + ".dump");
389 ret = new PrintStream (new FileOutputStream (file), false);
390 dbIdToOutputStream.put(dbId, ret);
391 Boolean dupSort = (Boolean ) dbIdToDupSort.get(dbId);
392 if (dupSort == null) {
393 dupSort = Boolean.valueOf(false);
394 }
395 printHeader(ret, dupSort.booleanValue(), formatUsingPrintable);
396 return ret;
397 } catch (IOException IOE) {
398 throw new DatabaseException(IOE);
399 }
400 }
401
402 private void closeOutputStreams() {
403
404 Iterator iter = dbIdToOutputStream.values().iterator();
405 while (iter.hasNext()) {
406 PrintStream s = (PrintStream ) iter.next();
407 s.println("DATA=END");
408 s.close();
409 }
410 }
411 }
412 |
Popular Tags |
Java API By Example, From Geeks To Geeks.