|
1
8
9 package com.sleepycat.je.util;
10
11 import java.io.File  ;
12 import java.io.PrintStream ;
13 import java.math.BigInteger ;
14 import java.text.NumberFormat ;
15 import java.util.Random ;
16
17 import com.sleepycat.je.Database;
18 import com.sleepycat.je.DatabaseConfig;
19 import com.sleepycat.je.DatabaseEntry;
20 import com.sleepycat.je.DatabaseException;
21 import com.sleepycat.je.Environment;
22 import com.sleepycat.je.EnvironmentConfig;
23 import com.sleepycat.je.EnvironmentStats;
24 import com.sleepycat.je.OperationStatus;
25 import com.sleepycat.je.dbi.MemoryBudget;
26 import com.sleepycat.je.utilint.CmdUtil;
27
28
73
74 public class DbCacheSize {
75
76 private static final NumberFormat INT_FORMAT =
77 NumberFormat.getIntegerInstance();
78
79 private static final String HEADER =
80 " Cache Size Btree Size Description\n" +
81 "-------------- -------------- -----------";
82 private static final int COLUMN_WIDTH = 14;
85 private static final int COLUMN_SEPARATOR = 2;
86
87 private long records;
88 private int keySize;
89 private int dataSize;
90 private int nodeMax;
91 private int density;
92 private long overhead;
93 private long minInBtreeSize;
94 private long maxInBtreeSize;
95 private long minInCacheSize;
96 private long maxInCacheSize;
97 private long maxInBtreeSizeWithData;
98 private long maxInCacheSizeWithData;
99 private long minInBtreeSizeWithData;
100 private long minInCacheSizeWithData;
101 private int nLevels = 1;
102
103 public DbCacheSize (long records,
104 int keySize,
105 int dataSize,
106 int nodeMax,
107 int density,
108 long overhead) {
109 this.records = records;
110 this.keySize = keySize;
111 this.dataSize = dataSize;
112 this.nodeMax = nodeMax;
113 this.density = density;
114 this.overhead = overhead;
115 }
116
117 public long getMinCacheSizeInternalNodesOnly() {
118 return minInCacheSize;
119 }
120
121 public long getMaxCacheSizeInternalNodesOnly() {
122 return maxInCacheSize;
123 }
124
125 public long getMinBtreeSizeInternalNodesOnly() {
126 return minInBtreeSize;
127 }
128
129 public long getMaxBtreeSizeInternalNodesOnly() {
130 return maxInBtreeSize;
131 }
132
133 public long getMinCacheSizeWithData() {
134 return minInCacheSizeWithData;
135 }
136
137 public long getMaxCacheSizeWithData() {
138 return maxInCacheSizeWithData;
139 }
140
141 public long getMinBtreeSizeWithData() {
142 return minInBtreeSizeWithData;
143 }
144
145 public long getMaxBtreeSizeWithData() {
146 return maxInBtreeSizeWithData;
147 }
148
149 public int getNLevels() {
150 return nLevels;
151 }
152
153 public static void main(String [] args) {
154
155 try {
156 long records = 0;
157 int keySize = 0;
158 int dataSize = 0;
159 int nodeMax = 128;
160 int density = 80;
161 long overhead = 0;
162 File measureDir = null;
163 boolean measureRandom = false;
164
165 for (int i = 0; i < args.length; i += 1) {
166 String name = args[i];
167 String val = null;
168 if (i < args.length - 1 && !args[i + 1].startsWith("-")) {
169 i += 1;
170 val = args[i];
171 }
172 if (name.equals("-records")) {
173 if (val == null) {
174 usage("No value after -records");
175 }
176 try {
177 records = Long.parseLong(val);
178 } catch (NumberFormatException e) {
179 usage(val + " is not a number");
180 }
181 if (records <= 0) {
182 usage(val + " is not a positive integer");
183 }
184 } else if (name.equals("-key")) {
185 if (val == null) {
186 usage("No value after -key");
187 }
188 try {
189 keySize = Integer.parseInt(val);
190 } catch (NumberFormatException e) {
191 usage(val + " is not a number");
192 }
193 if (keySize <= 0) {
194 usage(val + " is not a positive integer");
195 }
196 } else if (name.equals("-data")) {
197 if (val == null) {
198 usage("No value after -data");
199 }
200 try {
201 dataSize = Integer.parseInt(val);
202 } catch (NumberFormatException e) {
203 usage(val + " is not a number");
204 }
205 if (dataSize <= 0) {
206 usage(val + " is not a positive integer");
207 }
208 } else if (name.equals("-nodemax")) {
209 if (val == null) {
210 usage("No value after -nodemax");
211 }
212 try {
213 nodeMax = Integer.parseInt(val);
214 } catch (NumberFormatException e) {
215 usage(val + " is not a number");
216 }
217 if (nodeMax <= 0) {
218 usage(val + " is not a positive integer");
219 }
220 } else if (name.equals("-density")) {
221 if (val == null) {
222 usage("No value after -density");
223 }
224 try {
225 density = Integer.parseInt(val);
226 } catch (NumberFormatException e) {
227 usage(val + " is not a number");
228 }
229 if (density < 1 || density > 100) {
230 usage(val + " is not betwen 1 and 100");
231 }
232 } else if (name.equals("-overhead")) {
233 if (val == null) {
234 usage("No value after -overhead");
235 }
236 try {
237 overhead = Long.parseLong(val);
238 } catch (NumberFormatException e) {
239 usage(val + " is not a number");
240 }
241 if (overhead < 0) {
242 usage(val + " is not a non-negative integer");
243 }
244 } else if (name.equals("-measure")) {
245 if (val == null) {
246 usage("No value after -measure");
247 }
248 measureDir = new File (val);
249 } else if (name.equals("-measurerandom")) {
250 measureRandom = true;
251 } else {
252 usage("Unknown arg: " + name);
253 }
254 }
255
256 if (records == 0) {
257 usage("-records not specified");
258 }
259
260 if (keySize == 0) {
261 usage("-key not specified");
262 }
263
264 DbCacheSize dbCacheSize = new DbCacheSize
265 (records, keySize, dataSize, nodeMax, density, overhead);
266 dbCacheSize.caclulateCacheSizes();
267 dbCacheSize.printCacheSizes(System.out);
268
269 if (measureDir != null) {
270 measure(System.out, measureDir, records, keySize, dataSize,
271 nodeMax, measureRandom);
272 }
273 } catch (Throwable e) {
274 e.printStackTrace(System.out);
275 }
276 }
277
278 private static void usage(String msg) {
279
280 if (msg != null) {
281 System.out.println(msg);
282 }
283
284 System.out.println
285 ("usage:" +
286 "\njava " + CmdUtil.getJavaCommand(DbCacheSize.class) +
287 "\n -records <count>" +
288 "\n # Total records (key/data pairs); required" +
289 "\n -key <bytes> " +
290 "\n # Average key bytes per record; required" +
291 "\n [-data <bytes>]" +
292 "\n # Average data bytes per record; if omitted no leaf" +
293 "\n # node sizes are included in the output" +
294 "\n [-nodemax <entries>]" +
295 "\n # Number of entries per Btree node; default: 128" +
296 "\n [-density <percentage>]" +
297 "\n # Percentage of node entries occupied; default: 80" +
298 "\n [-overhead <bytes>]" +
299 "\n # Overhead of non-Btree objects (log buffers, locks," +
300 "\n # etc); default: 10% of total cache size" +
301 "\n [-measure <environmentHomeDirectory>]" +
302 "\n # An empty directory used to write a database to find" +
303 "\n # the actual cache size; default: do not measure" +
304 "\n [-measurerandom" +
305 "\n # With -measure insert randomly generated keys;" +
306 "\n # default: insert sequential keys");
307
308 System.exit(2);
309 }
310
311 private void caclulateCacheSizes() {
312 int nodeAvg = (nodeMax * density) / 100;
313 long nBinEntries = (records * nodeMax) / nodeAvg;
314 long nBinNodes = (nBinEntries + nodeMax - 1) / nodeMax;
315
316 long nInNodes = 0;
317 long lnSize = 0;
318
319 for (long n = nBinNodes; n > 0; n /= nodeMax) {
320 nInNodes += n;
321 nLevels += 1;
322 }
323
324 minInBtreeSize = nInNodes *
325 calcInSize(nodeMax, nodeAvg, keySize, true);
326 maxInBtreeSize = nInNodes *
327 calcInSize(nodeMax, nodeAvg, keySize, false);
328 minInCacheSize = calculateOverhead(minInBtreeSize, overhead);
329 maxInCacheSize = calculateOverhead(maxInBtreeSize, overhead);
330
331 if (dataSize > 0) {
332 lnSize = records * calcLnSize(dataSize);
333 }
334
335 maxInBtreeSizeWithData = maxInBtreeSize + lnSize;
336 maxInCacheSizeWithData = calculateOverhead(maxInBtreeSizeWithData,
337 overhead);
338 minInBtreeSizeWithData = minInBtreeSize + lnSize;
339 minInCacheSizeWithData = calculateOverhead(minInBtreeSizeWithData,
340 overhead);
341 }
342
343 private void printCacheSizes(PrintStream out) {
344
345 out.println("Inputs:" +
346 " records=" + records +
347 " keySize=" + keySize +
348 " dataSize=" + dataSize +
349 " nodeMax=" + nodeMax +
350 " density=" + density + '%' +
351 " overhead=" + ((overhead > 0) ? overhead : 10) + "%");
352
353 out.println();
354 out.println(HEADER);
355 out.println(line(minInBtreeSize, minInCacheSize,
356 "Minimum, internal nodes only"));
357 out.println(line(maxInBtreeSize, maxInCacheSize,
358 "Maximum, internal nodes only"));
359 if (dataSize > 0) {
360 out.println(line(minInBtreeSizeWithData,
361 minInCacheSizeWithData,
362 "Minimum, internal nodes and leaf nodes"));
363 out.println(line(maxInBtreeSizeWithData,
364 maxInCacheSizeWithData,
365 "Maximum, internal nodes and leaf nodes"));
366 } else {
367 out.println("\nTo get leaf node sizing specify -data");
368 }
369
370 out.println("\nBtree levels: " + nLevels);
371 }
372
373 private int calcInSize(int nodeMax,
374 int nodeAvg,
375 int keySize,
376 boolean lsnCompression) {
377
378
379 int size = MemoryBudget.IN_FIXED_OVERHEAD;
380
381
382 size += MemoryBudget.byteArraySize(nodeMax) +
383 (nodeMax * (2 * MemoryBudget.ARRAY_ITEM_OVERHEAD));
384
385
386 if (lsnCompression) {
387 size += MemoryBudget.byteArraySize(nodeMax * 2);
388 } else {
389 size += MemoryBudget.BYTE_ARRAY_OVERHEAD +
390 (nodeMax * MemoryBudget.LONG_OVERHEAD);
391 }
392
393
394 size += (nodeAvg + 1) * MemoryBudget.byteArraySize(keySize);
395
396 return size;
397 }
398
399 private int calcLnSize(int dataSize) {
400
401 return MemoryBudget.LN_OVERHEAD +
402 MemoryBudget.byteArraySize(dataSize);
403 }
404
405 private long calculateOverhead(long btreeSize, long overhead) {
406 long cacheSize;
407 if (overhead == 0) {
408 cacheSize = (100 * btreeSize) / 90;
409 } else {
410 cacheSize = btreeSize + overhead;
411 }
412 return cacheSize;
413 }
414
415 private String line(long btreeSize,
416 long cacheSize,
417 String comment) {
418
419 StringBuffer buf = new StringBuffer (100);
420
421 column(buf, INT_FORMAT.format(cacheSize));
422 column(buf, INT_FORMAT.format(btreeSize));
423 column(buf, comment);
424
425 return buf.toString();
426 }
427
428 private void column(StringBuffer buf, String str) {
429
430 int start = buf.length();
431
432 while (buf.length() - start + str.length() < COLUMN_WIDTH) {
433 buf.append(' ');
434 }
435
436 buf.append(str);
437
438 for (int i = 0; i < COLUMN_SEPARATOR; i += 1) {
439 buf.append(' ');
440 }
441 }
442
443 private static void measure(PrintStream out,
444 File dir,
445 long records,
446 int keySize,
447 int dataSize,
448 int nodeMax,
449 boolean randomKeys)
450 throws DatabaseException {
451
452 String [] fileNames = dir.list();
453 if (fileNames != null && fileNames.length > 0) {
454 usage("Directory is not empty: " + dir);
455 }
456
457 Environment env = openEnvironment(dir, true);
458 Database db = openDatabase(env, nodeMax, true);
459
460 try {
461 out.println("\nMeasuring with cache size: " +
462 INT_FORMAT.format(env.getConfig().getCacheSize()));
463 insertRecords(out, env, db, records, keySize, dataSize, randomKeys);
464 printStats(out, env,
465 "Stats for internal and leaf nodes (after insert)");
466
467 db.close();
468 env.close();
469 env = openEnvironment(dir, false);
470 db = openDatabase(env, nodeMax, false);
471
472 out.println("\nPreloading with cache size: " +
473 INT_FORMAT.format(env.getConfig().getCacheSize()));
474 preloadRecords(out, db);
475 printStats(out, env,
476 "Stats for internal nodes only (after preload)");
477 } finally {
478 try {
479 db.close();
480 env.close();
481 } catch (Exception e) {
482 out.println("During close: " + e);
483 }
484 }
485 }
486
487 private static Environment openEnvironment(File dir, boolean allowCreate)
488 throws DatabaseException {
489
490 EnvironmentConfig envConfig = new EnvironmentConfig();
491 envConfig.setAllowCreate(allowCreate);
492 envConfig.setCachePercent(90);
493 return new Environment(dir, envConfig);
494 }
495
496 private static Database openDatabase(Environment env, int nodeMax,
497 boolean allowCreate)
498 throws DatabaseException {
499
500 DatabaseConfig dbConfig = new DatabaseConfig();
501 dbConfig.setAllowCreate(allowCreate);
502 dbConfig.setNodeMaxEntries(nodeMax);
503 return env.openDatabase(null, "foo", dbConfig);
504 }
505
506 private static void insertRecords(PrintStream out,
507 Environment env,
508 Database db,
509 long records,
510 int keySize,
511 int dataSize,
512 boolean randomKeys)
513 throws DatabaseException {
514
515 DatabaseEntry key = new DatabaseEntry();
516 DatabaseEntry data = new DatabaseEntry(new byte[dataSize]);
517 BigInteger bigInt = BigInteger.ZERO;
518 Random rnd = new Random (123);
519
520 for (int i = 0; i < records; i += 1) {
521
522 if (randomKeys) {
523 byte[] a = new byte[keySize];
524 rnd.nextBytes(a);
525 key.setData(a);
526 } else {
527 bigInt = bigInt.add(BigInteger.ONE);
528 byte[] a = bigInt.toByteArray();
529 if (a.length < keySize) {
530 byte[] a2 = new byte[keySize];
531 System.arraycopy(a, 0, a2, a2.length - a.length, a.length);
532 a = a2;
533 } else if (a.length > keySize) {
534 out.println("*** Key doesn't fit value=" + bigInt +
535 " byte length=" + a.length);
536 return;
537 }
538 key.setData(a);
539 }
540
541 OperationStatus status = db.putNoOverwrite(null, key, data);
542 if (status == OperationStatus.KEYEXIST && randomKeys) {
543 i -= 1;
544 out.println("Random key already exists -- retrying");
545 continue;
546 }
547 if (status != OperationStatus.SUCCESS) {
548 out.println("*** " + status);
549 return;
550 }
551
552 if (i % 10000 == 0) {
553 EnvironmentStats stats = env.getStats(null);
554 if (stats.getNNodesScanned() > 0) {
555 out.println("*** Ran out of cache memory at record " + i +
556 " -- try increasing the Java heap size ***");
557 return;
558 }
559 out.print(".");
560 out.flush();
561 }
562 }
563 }
564
565 private static void preloadRecords(final PrintStream out,
566 final Database db)
567 throws DatabaseException {
568
569 Thread thread = new Thread () {
570 public void run() {
571 while (true) {
572 try {
573 out.print(".");
574 out.flush();
575 Thread.sleep(5 * 1000);
576 } catch (InterruptedException e) {
577 break;
578 }
579 }
580 }
581 };
582 thread.start();
583 db.preload(0);
584 thread.interrupt();
585 try {
586 thread.join();
587 } catch (InterruptedException e) {
588 e.printStackTrace(out);
589 }
590 }
591
592 private static void printStats(PrintStream out,
593 Environment env,
594 String msg)
595 throws DatabaseException {
596
597 out.println();
598 out.println(msg + ':');
599
600 EnvironmentStats stats = env.getStats(null);
601
602 out.println("CacheSize=" +
603 INT_FORMAT.format(stats.getCacheTotalBytes()) +
604 " BtreeSize=" +
605 INT_FORMAT.format(stats.getCacheDataBytes()));
606
607 if (stats.getNNodesScanned() > 0) {
608 out.println("*** All records did not fit in the cache ***");
609 }
610 }
611 }
612 |
Popular Tags |
Java API By Example, From Geeks To Geeks.