1 package com.quadcap.sql.file; 2 3 40 41 import java.io.ByteArrayOutputStream ; 42 import java.io.File ; 43 import java.io.FileInputStream ; 44 import java.io.IOException ; 45 import java.io.InputStream ; 46 import java.io.OutputStream ; 47 import java.io.PrintWriter ; 48 49 import java.util.Properties ; 50 51 import com.quadcap.util.ConfigNumber; 52 import com.quadcap.util.Debug; 53 import com.quadcap.util.Util; 54 55 import com.quadcap.util.collections.LongMap; 56 57 import com.quadcap.crypto.KeyFactory; 58 import com.quadcap.crypto.SymmetricKey; 59 60 import com.quadcap.io.MutableByteArrayInputStream; 61 62 import com.quadcap.sql.io.ObjectInputStream; 63 import com.quadcap.sql.io.ObjectOutputStream; 64 65 import com.quadcap.sql.Version; 66 67 74 75 public class BlockFile implements PageManager, SegmentManager { 76 77 BlockStore store; 78 79 80 BlockCache cache; 81 82 83 LongMap memoryObjects = null; 84 long memObjectCount = 100000; 85 boolean inMemory; 86 87 static final int REF_SIZE = 8; 88 89 static final int MIN_SUBPAGE = 64; 90 91 static final int MAX_PAGESHIFT = 16; 92 93 int maxPageShift = 0; 95 static final int oBLOCKSIZE = 4; 102 103 static final int oFREELIST = 8; 105 106 static final int oLASTBLOCK = 16; 108 109 static final int oSTREAM_START = 24; 111 static final int MAX_STREAM = 4; 112 113 static final int oHASH_PASSWD_start 115 = oSTREAM_START + (REF_SIZE * MAX_STREAM); 116 static final int oHASH_PASSWD_len = 20; 117 static final int oSUBPAGE_ROOT 119 = oHASH_PASSWD_start + oHASH_PASSWD_len; 120 static final int oHEADER_SIZE = (oSUBPAGE_ROOT + 121 (MAX_PAGESHIFT * REF_SIZE)); 122 123 int blockSize; 124 125 126 long lastBlock = 0; 127 128 boolean readOnly; 129 130 PageManager[] pageManagers = new PageManager[MAX_PAGESHIFT]; 131 Object fileLock = new Object (); 132 133 MutableByteArrayInputStream mbis = new MutableByteArrayInputStream(); 134 ObjectInputStream ois = new ObjectInputStream(mbis); 135 ByteArrayOutputStream bos = new ByteArrayOutputStream (); 136 ObjectOutputStream oos; 137 138 147 public BlockFile(String filename, String mode, 148 Properties props, 149 int blocksize, 150 int cacheSize) 151 throws IOException 152 { 153 if (Trace.bit(4)) { 155 Debug.println("BlockFile(" + filename + ", " + mode + 156 ", blockSize = " + blocksize + 157 ", cacheSize = " + cacheSize + ")"); 158 } 159 this.inMemory = 161 props.getProperty("memoryOnly", "").equalsIgnoreCase("true"); 162 pageManagers[0] = this; 163 int psize = blocksize; 164 maxPageShift = 0; 165 while (psize > MIN_SUBPAGE) { 166 maxPageShift++; 167 psize >>= 1; 168 } 169 this.readOnly = mode.equalsIgnoreCase("r"); 170 this.store = makeStore(props, filename, blocksize, mode); 171 this.blockSize = store.blockSize(); 172 this.cache = new BlockCache(); 173 this.oos = new ObjectOutputStream(bos); 174 if (inMemory) { 175 this.memoryObjects = new LongMap(8888); 176 } 177 cache.setLock(fileLock); 178 cache.setReadOnly(readOnly); 179 cache.init(store, cacheSize); 180 Block root = getBlock(0); 181 try { 182 this.lastBlock = root.readLong(oLASTBLOCK); 183 } finally { 184 root.decrRefCount(); 185 } 186 } 187 188 public String signature() throws IOException { 190 StringBuffer sb = new StringBuffer ("{"); 191 for (long blk = 0; blk <= lastBlock; blk++) { 192 if (blk > 0) sb.append(","); 193 Block b = getBlock(blk); 194 try { 195 sb.append(b.signature()); 196 } finally { 197 b.decrRefCount(); 198 } 199 } 200 sb.append("}"); 201 return sb.toString(); 202 } 203 205 protected BlockStore makeStore(Properties props, String filename, 206 int blockSize, String mode) 207 throws IOException 208 { 209 BlockStore s; 210 File f = null; 211 boolean enc = false; 212 boolean nullStore = 213 props.getProperty("cacheOnly", "").equalsIgnoreCase("true"); 214 if (inMemory || nullStore) { 215 if (inMemory) { 216 s = new MemoryBlockStore(); 217 } else { 218 s = new NullStore(); 219 } 220 } else { 221 filename = new File (filename).getAbsolutePath(); 222 f = new File (filename); 223 boolean exists = f.exists(); 224 if (!exists && readOnly) { 225 throw new IOException ("Not found: " + filename); 226 } 227 String u = props.getProperty("user", 228 props.getProperty("jdbc.user", "")); 229 String p = props.getProperty("passwd", 230 props.getProperty("jdbc.passwd", "")); 231 247 boolean encrypt = 248 props.getProperty("encrypt", "").equalsIgnoreCase("true"); 249 enc = (exists && isEncrypted(f)) || (!exists && encrypt); 250 if (enc) { 251 s = new EncryptedBlockStore(); 252 } else { 253 s = new BlockStore(); 254 } 255 } 256 s.init(f, mode, blockSize, fileLock); 257 if (enc) { 258 String u = props.getProperty("user", 259 props.getProperty("jdbc.user", "")); 260 String p = props.getProperty("passwd", 261 props.getProperty("jdbc.passwd", "")); 262 SymmetricKey key = KeyFactory.createSymmetricKey(u + "/" + p); 263 s.setKey(key); 264 } 265 return s; 266 } 267 268 private boolean isEncrypted(File f) throws IOException { 269 FileInputStream fi = new FileInputStream (f); 270 try { 271 return fi.read() == 0x0e; 272 } finally { 273 fi.close(); 274 } 275 } 276 277 public long getUserBlock(int stream) throws IOException { 278 synchronized (fileLock) { 279 if (stream > MAX_STREAM) { 280 throw new IOException ("Invalid stream: " + stream); 281 } 282 Block root = getBlock(0); 283 long blockNum; 284 int offset = oSTREAM_START + (stream * REF_SIZE); 285 try { 286 blockNum = root.readLong(offset); 287 if (blockNum == 0) { 288 blockNum = newPage(); 289 root.writeLong(offset, blockNum); 290 } 291 } finally { 292 root.decrRefCount(); 293 } 294 return blockNum; 295 } 296 } 297 298 301 public void close() throws IOException { 302 synchronized (fileLock) { 303 flush(false); 304 store.close(); 305 } 306 } 307 308 public final void flush(boolean fastSync) throws IOException { 309 synchronized (fileLock) { 310 if (Trace.bit(24)) { 312 Debug.println(toString() + ".flush(" + fastSync + ")"); 313 } 314 if (!readOnly) { 316 cache.flush(); 317 if (!fastSync) { 318 store.flush(); 319 } 320 } 321 } 322 } 323 324 public String getName() { 325 return store.getName(); 326 } 327 328 public void setLog(Log log) { 329 store.setLog(log); 330 } 331 332 Log getLog() { 333 return store.getLog(); 334 } 335 336 344 public Page getPage(long i) throws IOException { 345 synchronized (fileLock) { 346 if (i > lastBlock) { 347 Debug.println(toString() + ".getPage(" + i + "): Bad Block " + 349 " vs lastBlock = " + lastBlock); 350 throw new IOException ("BlockFile(" + store + 352 "), Bad block: " + i + " (" + 353 SubPageManager.toString(i) + ")"); 354 } 355 return (Page)cache.getCacheable(i); 356 } 357 } 358 359 public Block getBlock(long i) throws IOException { 360 return (Block)getPage(i); 361 } 362 363 public void restoreBlock(long b, byte[] buf, int off) throws IOException { 364 ensureLastBlock(b); 365 store.restore(b, buf, off); 366 } 367 368 final void ensureLastBlock(long i) throws IOException { 369 if (i > lastBlock) { 370 lastBlock = i; 371 Block root = getBlock(0); 372 try { 373 root.writeLong(oLASTBLOCK, i); 374 } finally { 375 root.decrRefCount(); 376 } 377 } 378 } 379 380 381 384 public final int getPageSize() { 385 return this.blockSize; 386 } 387 388 391 public RandomAccess getStream(long blockRef) throws IOException { 392 PageManager p = getPageManagerForPage(blockRef); 393 return new BlockAccess(p, blockRef); 394 } 395 396 400 public long newPage() throws IOException { 401 synchronized (fileLock) { 402 Block root = getBlock(0); 403 long ref; 404 try { 405 ref = root.readLong(oFREELIST); 406 if (ref < 0 || ref > lastBlock) { 407 throw new IOException (this + "newPage(), bad ref: " + 408 ref + ", lastBlock = " + 409 lastBlock); 410 } 411 if (ref != 0) { 412 Block newPage = getBlock(ref); 413 try { 414 root.writeLong(oFREELIST, newPage.readLong(0)); 415 newPage.clear(); 417 } finally { 418 newPage.decrRefCount(); 419 } 420 } else { 421 ref = root.readLong(oLASTBLOCK); 422 Block newPage = getBlock(ref); 423 try { 424 newPage.clear(); 425 } finally { 426 newPage.decrRefCount(); 427 } 428 lastBlock = ref + 1; 429 root.writeLong(oLASTBLOCK, lastBlock); 430 } 431 } finally { 432 root.decrRefCount(); 433 } 434 if (Trace.bit(5)) { 436 Debug.println(toString() + ".newPage() = " + ref + ""); 437 } 438 return ref; 440 } 441 } 442 443 448 public void freePage(long ref) throws IOException { 449 PageManager p = getPageManagerForPage(ref); 450 if (p != this) { 451 p.freePage(ref); 452 } else synchronized (fileLock) { 453 if (Trace.bit(5)) { 455 Debug.println(toString() + ".freePage(" + ref + ")"); 456 } 457 if (ref <= 0 || ref > lastBlock) { 459 throw new IOException ("bad ref"); 460 } 461 Block blk = getBlock(ref); 462 try { 463 blk.getDataAndReset(); 464 465 Block root = getBlock(0); 466 try { 467 long freeList = root.readLong(oFREELIST); 468 blk.writeLong(0, freeList); 469 root.writeLong(oFREELIST, ref); 470 } finally { 471 root.decrRefCount(); 472 } 473 } finally { 474 blk.decrRefCount(); 475 } 476 } 477 } 478 479 static int getPageOffset(PageManager pm, long ref) { 480 int psize = pm.getPageSize(); 481 int pnum = (int)(SubPageManager.fExt(ref, SubPageManager.PAGE_NUM)); 482 return pnum * psize; 483 } 484 485 static long getPageBlock(long ref) { 486 return SubPageManager.pageBlock(ref); 487 } 488 489 static final boolean quick = true; 490 491 498 public long putBytes(byte[] buf) throws IOException { 499 PageManager pm = getPageManagerForLen(buf.length); 500 long seg = pm.newPage(); 501 if (quick && (buf.length + 8) < pm.getPageSize()) { 502 int off = getPageOffset(pm, seg); 503 long blk = getPageBlock(seg); 504 synchronized (fileLock) { 505 Block b = getBlock(blk); 506 try { 507 b.writeLong(off, buf.length); 508 b.write(off+8, buf, 0, buf.length); 509 } finally { 510 b.decrRefCount(); 511 } 512 } 513 } else { 514 synchronized (fileLock) { 515 ba.init(pm, seg); 516 ba.resize(buf.length); 517 ba.write(0, buf, 0, buf.length); 518 } 519 } 520 return seg; 524 } 525 526 BlockAccess ba = new BlockAccess(); 527 528 534 public byte[] getBytes(long seg) throws IOException { 535 PageManager pm = getPageManagerForPage(seg); 536 byte[] buf = null; 537 int off = getPageOffset(pm, seg); 538 long blk = getPageBlock(seg); 539 synchronized (fileLock) { 540 if (quick) { 541 Block b = getBlock(blk); 542 try { 543 int len = (int)b.readLong(off); 544 if (len + 8 < pm.getPageSize()) { 545 buf = new byte[len]; 546 b.read(off+8, buf, 0, len); 547 } 548 } finally { 549 b.decrRefCount(); 550 } 551 } 552 if (buf == null) { 553 ba.init(pm, seg); 554 buf = new byte[(int)(ba.size())]; 555 ba.read(0, buf, 0, buf.length); 556 } 557 } 558 return buf; 562 } 563 564 565 571 public void updateBytes(long seg, byte[] buf) throws IOException { 572 PageManager pm = getPageManagerForPage(seg); 576 synchronized (fileLock) { 577 if (quick && (buf.length + 8) < pm.getPageSize()) { 578 Block b = getBlock(getPageBlock(seg)); 579 try { 580 int off = getPageOffset(pm, seg); 581 long oldLen = b.readLong(off); 582 if ((oldLen + 8) < pm.getPageSize()) { 583 b.writeLong(off, buf.length); 584 b.write(off+8, buf, 0, buf.length); 585 return; 586 } 587 } finally { 588 b.decrRefCount(); 589 } 590 } 591 592 ba.init(pm, seg); 593 if (ba.size() != buf.length) { 594 ba.resize(buf.length); 595 } 596 ba.write(0, buf, 0, buf.length); 597 } 598 } 599 600 609 public void freeStream(long page) throws IOException { 610 synchronized (fileLock) { 611 getStream(page).resize(0); 612 freePage(page); 613 } 614 } 615 public void freeSegment(long seg) throws IOException { 616 freeStream(seg); 617 } 618 619 628 public RandomAccessInputStream getInputStream(long block) 629 throws IOException 630 { 631 return new RandomAccessInputStream(getStream(block)); 632 } 633 634 643 public RandomAccessOutputStream getOutputStream(long block) 644 throws IOException 645 { 646 return new RandomAccessOutputStream(getStream(block)); 647 } 648 649 652 public PageManager getPageManager(int i) throws IOException { 653 if (i < 0 || i > maxPageShift) { 654 throw new IOException ("Bad page shift: " + i); 655 } 656 PageManager pm = pageManagers[i]; 657 if (pm == null) { 658 pageManagers[i] = pm = new SubPageManager(this, i, 0); 659 } 660 return pm; 661 } 662 663 public PageManager getPageManagerForPage(long page) throws IOException { 664 return getPageManager(SubPageManager.pageShift(page)); 665 } 666 667 public PageManager getPageManagerForLen(long len) throws IOException { 668 int pageShift = 0; 669 int siz = blockSize >> 1; 670 while (len < siz && pageShift < maxPageShift-1) { 671 siz >>= 1; 672 pageShift++; 673 } 674 if (pageShift > 0) { 675 pageShift--; 676 } 677 return getPageManager(pageShift); 678 } 679 680 public void revert() throws IOException { 681 cache.revert(); 682 Block root = getBlock(0); 683 try { 684 this.lastBlock = root.readLong(oLASTBLOCK); 685 } finally { 686 root.decrRefCount(); 687 } 688 store.setLength(this.lastBlock * blockSize); 689 } 690 691 public void showCache(PrintWriter os) { 692 cache.show(os); 693 } 694 695 public String toString() { 696 return "BlockFile(" + store + ")"; 697 } 698 699 public void clearModified() throws IOException { 700 store.clearModified(); 701 } 702 703 public long getSize() { 704 return lastBlock * blockSize; 705 } 706 707 public Object getLock() { 708 return fileLock; 709 } 710 711 public int getBlockSize() { 712 return blockSize; 713 } 714 715 716 717 723 public Object getObject(long ref) throws IOException { 724 Object ret = null; 725 synchronized (fileLock) { 726 if (inMemory) { 727 ret = memoryObjects.get(ref); 728 } else { 729 try { 730 byte[] buf = getBytes(ref); 731 mbis.reset(buf); 732 ret = ois.readObject(); 733 } catch (ClassNotFoundException e) { 734 throw new DatafileException(e); 735 } 736 } 737 } 738 return ret; 740 } 741 742 743 749 public long putObject(Object obj) throws IOException { 750 synchronized (fileLock) { 751 if (inMemory) { 752 memoryObjects.put(++memObjectCount, obj); 753 return memObjectCount; 755 } 756 bos.reset(); 757 oos.writeObject(obj); 758 oos.flush(); 759 byte[] buf = bos.toByteArray(); 760 long ret = putBytes(buf); 761 return ret; 762 } 763 } 764 765 772 public void updateObject(long seg, Object obj) throws IOException { 773 synchronized (fileLock) { 775 if (inMemory) { 776 memoryObjects.put(seg, obj); 777 } else { 778 bos.reset(); 779 oos.writeObject(obj); 780 oos.flush(); 781 byte[] buf = bos.toByteArray(); 782 updateBytes(seg, buf); 783 } 784 } 785 } 786 787 793 public void removeObject(long ref) throws IOException { 794 synchronized (fileLock) { 796 if (inMemory) { 797 memoryObjects.remove(ref); 798 } else { 799 freeSegment(ref); 800 } 801 } 802 } 803 804 805 806 public void dump() throws IOException { 808 Block root = getBlock(0); 809 try { 810 Debug.println("Magic: " + 811 Integer.toHexString(root.readShort(0))); 812 Debug.println("Major version: " + root.readByte(2)); 813 Debug.println("Minor version: " + root.readByte(3)); 814 Debug.println("Block Size: " + root.readInt(oBLOCKSIZE)); 815 Debug.println("FreeList: " + root.readLong(oFREELIST)); 816 java.util.BitSet free = new java.util.BitSet (); 817 for (long ref = root.readLong(oFREELIST); ref != 0; ) { 818 free.set((int)ref); 819 Block newPage = getBlock(ref); 820 try { 821 ref = newPage.readLong(0); 822 } finally { 823 newPage.decrRefCount(); 824 } 825 } 826 int first = -2; 827 int last = -2; 828 for (int i = free.nextSetBit(0); i >= 0; i = free.nextSetBit(i+1)) { 829 if (i != last+1) { 830 if (first >= 0) { 831 if (first == last) { 832 Debug.println("Free: " + first); 833 } else { 834 Debug.println("Free: " + first + " - " + last); 835 } 836 } 837 first = i; 838 last = i; 839 } 840 } 841 if (first >= 0) { 842 if (first == last) { 843 Debug.println("Free: " + first); 844 } else { 845 Debug.println("Free: " + first + " - " + last); 846 } 847 } 848 long lastBlock_ = root.readLong(oLASTBLOCK); 849 Debug.println("Last Block: " + root.readLong(oLASTBLOCK)); 850 long waste = free.cardinality() * blockSize; 851 long total = lastBlock_ * blockSize; 852 double pct = 100 * ((double)waste) / ((double)total); 853 String pcts = java.text.NumberFormat.getInstance().format(pct); 854 Debug.println("Fragmentation: " + free.cardinality() + " of " + 855 lastBlock_ + " blocks = " + pcts + "% waste"); 856 Debug.println("Last Block: " + root.readLong(oLASTBLOCK)); 857 for (int i = 0; i < MAX_STREAM; i++) { 858 Debug.println("Stream " + i + ": " + 859 root.readLong(oSTREAM_START + i*REF_SIZE)); 860 } 861 byte[] hp = new byte[oHASH_PASSWD_len]; 862 root.read(oHASH_PASSWD_start, hp, 0, oHASH_PASSWD_len); 863 Debug.println("Hash Passwd: " + Util.hexBytes(hp)); 864 for (int i = 0; i < 16; i++) { 865 Debug.println("PM " + i + " root: " + 866 SubPageManager.toString(root.readLong(oSUBPAGE_ROOT + i * REF_SIZE))); 867 } 868 } finally { 869 root.decrRefCount(); 870 } 871 } 872 public static void main(String [] args) { 873 try { 874 BlockFile f = new BlockFile(args[0], "r", 875 new Properties (), 876 8192, 256); 877 f.dump(); 878 } catch (Throwable t) { 879 Debug.print(t); 880 } 881 } 882 883 } 885 | Popular Tags |