1 46 50 package org.mr.core.persistent.file; 51 52 import java.io.File ; 53 54 import java.io.IOException ; 55 import java.io.RandomAccessFile ; 56 import java.nio.ByteBuffer ; 57 import java.nio.MappedByteBuffer ; 58 import java.nio.channels.FileChannel ; 59 import java.nio.channels.FileLock ; 60 61 import org.apache.commons.logging.Log; 62 import org.apache.commons.logging.LogFactory; 63 import org.mr.core.persistent.PersistentConst; 64 import org.mr.core.util.byteable.ByteBufferFactory; 65 66 67 80 public class MantaFatFile { 81 82 public static final int MAX_FAT_FILE_SIZE = Integer.MAX_VALUE; 84 public static final int CLUSTER_SIZE =1024; 87 public static final int MAX_FILES_IN_FAT = 10000; 89 public static final int BYTES_TO_INDEX_CLUSTER = 4; 91 public static final int BYTES_DATA_LENGTH_IN_CLUSTER =2; 93 public static final int FILE_TABLE_SIZE = MAX_FILES_IN_FAT*BYTES_TO_INDEX_CLUSTER; 95 public static final int FAT_METADATA_SIZE = 1024; 97 98 private static final int ROOT_CULSTER_INDEX =FAT_METADATA_SIZE +FILE_TABLE_SIZE; 100 private static final int NULL_CLUSTER_INDEX = -1; 102 103 public static final int MAX_NUMBER_OF_CLUSTERS = (MAX_FAT_FILE_SIZE-ROOT_CULSTER_INDEX)/CLUSTER_SIZE; 105 public static final int INITIAL_CULSTER_CAPACITY = 500; 107 public static final int INCREMENT_CULSTER_VALUE =5000; 109 112 private static final int CLUSTER_NEXT_MD_OFFSET =0; 114 private static final int CLUSTER_PREVIOUS_MD_OFFSET =CLUSTER_NEXT_MD_OFFSET+BYTES_TO_INDEX_CLUSTER; 116 private static final int CLUSTER_DATA_LENGTH_MD_OFFSET = CLUSTER_PREVIOUS_MD_OFFSET+BYTES_TO_INDEX_CLUSTER; 119 private static final int CLUSTER_DATA_OFFSET =CLUSTER_DATA_LENGTH_MD_OFFSET+BYTES_DATA_LENGTH_IN_CLUSTER; 121 private static final int MAX_DATA_SIZE_IN_CLUSTER =CLUSTER_SIZE -((BYTES_TO_INDEX_CLUSTER*2)+BYTES_DATA_LENGTH_IN_CLUSTER); 123 124 127 private static final int FAT_TABLE_OFFSET = 0; 128 private static final int FAT_METADATA_OFFSET = FAT_TABLE_OFFSET+FILE_TABLE_SIZE; 129 130 133 private static final int METADATA_FILE_COUNT_OFFSET = 0; 134 private static final int METADATA_FREE_CLUSTER_COUNT_OFFSET = METADATA_FILE_COUNT_OFFSET+4; 135 private static final int METADATA_ASSIGNED_CLUSTER_COUNT_OFFSET = METADATA_FREE_CLUSTER_COUNT_OFFSET+4; 136 private static final int METADATA_FAT_FILE_SIZE_OFFSET = METADATA_ASSIGNED_CLUSTER_COUNT_OFFSET+4; 137 138 139 private int fileInFatCount = 0; 141 private int freeClustersCount = 0; 143 private int assignedClusterCount = 1; 145 private int currentFileSize =0; 147 148 private File physicalFile; 149 private RandomAccessFile randomAccessFile; 150 151 private FileLock lock; 152 private MappedByteBuffer fatBuff; 153 private String fatFilePath; 154 private static Log log; 155 156 157 160 MantaFatFile(String fatFilePath) { 161 log=LogFactory.getLog("MantaFatFile"); 162 try { 163 if(!recoverFromFile(fatFilePath)) 164 createNewFatFile(fatFilePath); 165 } catch (IOException e) { 166 log.error("problem creating new fat file",e ); 167 } 168 this.fatFilePath = fatFilePath; 169 170 } 171 172 175 public synchronized int[] getFileList(){ 176 int[] list = new int[fileInFatCount]; 177 int j = 0; 178 for(int i= 1 ; i < MAX_FILES_IN_FAT && j<list.length ; i++){ 179 if(fileExit(i)) 180 list[j++] =i ; 181 } 182 int[] temp; 183 if( j < fileInFatCount){ 184 temp = new int[j]; 186 for (int i = 0; i < j; i++) { 187 temp[i] = list[i]; 188 } 189 list = temp; 190 saveFilesInFatCount(j); 191 } 192 return list; 193 } 194 195 202 public synchronized void save(int fileName , ByteBuffer fileContent) throws IOException { 203 204 206 checkValidFileName(fileName); 207 if(fileExit(fileName)){ 208 delete(fileName); 209 } 210 211 int fileSize = fileContent.limit(); 212 int numOfClustersNeeded = (int)Math.ceil((double)fileSize/MAX_DATA_SIZE_IN_CLUSTER); 213 int clusterIndex = assignCluster(); 214 saveFileLength(clusterIndex ,fileSize ); 215 writeDataToCluster(clusterIndex , fileContent); 216 saveEntryInFileTable(fileName ,clusterIndex ); 217 int nextClusterIndex = NULL_CLUSTER_INDEX; 218 for (int i = 1; i < numOfClustersNeeded; i++) { 219 nextClusterIndex =assignCluster(); 220 writeDataToCluster(nextClusterIndex , fileContent); 221 linkClusters(clusterIndex ,nextClusterIndex ); 222 clusterIndex = nextClusterIndex; 223 } 224 fileInFatCount++; 225 saveFilesInFatCount(fileInFatCount); 226 if(MantaFileManager.forceEveryEntry){ 227 fatBuff.force(); 228 } 229 230 231 } 233 234 239 public synchronized boolean fileExit(int fileName){ 240 checkValidFileName(fileName); 241 int rootCluster = getRootClusterFromFileTable(fileName); 242 return rootCluster != NULL_CLUSTER_INDEX; 243 } 244 245 250 public synchronized boolean delete(int fileName){ 251 checkValidFileName(fileName); 252 int fileRootCluster = deleteEntryInFileTable(fileName); 254 if(fileRootCluster == NULL_CLUSTER_INDEX) 255 return false; 256 int fileSize = getFileLength(fileRootCluster); 257 int numOfClustersNeeded = (int)Math.ceil((double)fileSize/MAX_DATA_SIZE_IN_CLUSTER); 258 int lastFreeCluster = getLastFreeClusterIndex(); 262 linkClusters(lastFreeCluster ,fileRootCluster ); 263 int newLastFreeCluster = fileRootCluster; 265 for (int i = 1; i < numOfClustersNeeded; i++) { 266 newLastFreeCluster = getNextClusterIndex(newLastFreeCluster); 267 } 268 setLastFreeClusterIndex(newLastFreeCluster); 269 270 freeClustersCount += numOfClustersNeeded; 271 saveFreeClustersCount(freeClustersCount); 272 assignedClusterCount -= numOfClustersNeeded; 273 saveAssignedClusterCount(assignedClusterCount); 274 fileInFatCount--; 275 saveFilesInFatCount(fileInFatCount); 276 return true; 277 } 278 279 284 public synchronized ByteBuffer load(int fileName){ 285 checkValidFileName(fileName); 286 ByteBuffer result = null; 287 int clusterIndex = getRootClusterFromFileTable(fileName); 288 if(clusterIndex == NULL_CLUSTER_INDEX) 290 return null; 291 int fileSize = getFileLength(clusterIndex); 292 293 ByteBufferFactory pool =PersistentConst.getPersistentByteBufferPool(); 294 if(pool != null){ 295 result = pool.getBuffer(fileSize ); 296 }else{ 297 result = ByteBuffer.allocate(fileSize); 298 } 299 300 fileSize -= readDataFromCluster( clusterIndex , result ); 301 while(fileSize>0){ 302 clusterIndex =getNextClusterIndex(clusterIndex); 303 fileSize -= readDataFromCluster( clusterIndex , result ); 304 } 305 result.flip(); 306 return result; 307 } 308 309 313 private void checkValidFileName(int fileName){ 314 if(fileName > MAX_FILES_IN_FAT || fileName<1){ 315 throw new IllegalArgumentException ("Invalid file name "+fileName+" fileName must be between 1 and "+MAX_FILES_IN_FAT); 316 } 317 } 318 319 320 323 private int readDataFromCluster(int clusterIndex, ByteBuffer result) { 324 int clusterDataLength = getClusterDataLength(clusterIndex); 325 setPosition(clusterIndex+CLUSTER_DATA_OFFSET); 326 for(int i = 0 ; i<clusterDataLength ; i++ ){ 327 result.put(fatBuff.get()); 328 } 329 return clusterDataLength; 330 331 } 332 333 336 private int getFileLength(int fileRootCluster){ 337 return getPreviousClusterIndex(fileRootCluster); 338 } 339 340 343 private void saveFileLength(int fileRootCluster, int fileSize){ 344 setPreviousClusterIndex(fileSize , fileRootCluster); 345 } 346 347 351 private void writeDataToCluster(int clusterIndex , ByteBuffer fileContent ){ 352 setPosition(clusterIndex+CLUSTER_DATA_OFFSET); 353 int dataInCluster = 0; 354 while(fileContent.hasRemaining() && dataInCluster<MAX_DATA_SIZE_IN_CLUSTER){ 355 fatBuff.put(fileContent.get()); 356 dataInCluster++; 357 } 358 setClusterDataLength(clusterIndex ,(short)dataInCluster ); 359 } 360 361 366 private void saveEntryInFileTable(int fileName ,int firstClusterIndex ){ 367 setPosition(FAT_TABLE_OFFSET+(fileName*BYTES_TO_INDEX_CLUSTER)); 368 fatBuff.putInt(firstClusterIndex); 369 } 370 371 376 private int getRootClusterFromFileTable(int fileName){ 377 setPosition(FAT_TABLE_OFFSET+(fileName*BYTES_TO_INDEX_CLUSTER)); 378 return fatBuff.getInt(); 379 380 } 381 386 private int deleteEntryInFileTable(int fileName){ 387 388 setPosition(FAT_TABLE_OFFSET+(fileName*BYTES_TO_INDEX_CLUSTER)); 389 int fileRootCluster = fatBuff.getInt(); 390 setPosition(FAT_TABLE_OFFSET+(fileName*BYTES_TO_INDEX_CLUSTER)); 391 fatBuff.putInt(NULL_CLUSTER_INDEX); 392 return fileRootCluster; 393 } 394 395 400 private int assignCluster() throws IOException { 401 402 int nextFreeCluster = getNextClusterIndex(ROOT_CULSTER_INDEX); 403 if( nextFreeCluster == NULL_CLUSTER_INDEX){ 404 if(freeClustersCount != 0 && log.isWarnEnabled()){ 405 log.warn("FAT netFreeCluster=null : freeClustersCount = "+freeClustersCount); 406 407 } 408 allocateFreeClusters(); 409 nextFreeCluster = getNextClusterIndex(ROOT_CULSTER_INDEX); 410 } 411 freeClustersCount--; 412 saveFreeClustersCount(freeClustersCount); 413 assignedClusterCount++; 414 saveAssignedClusterCount(assignedClusterCount); 415 int newNextFree = getNextClusterIndex(nextFreeCluster); 417 if(newNextFree != NULL_CLUSTER_INDEX){ 418 linkClusters(ROOT_CULSTER_INDEX ,newNextFree ); 420 }else{ 421 setNextClusterIndex(ROOT_CULSTER_INDEX , NULL_CLUSTER_INDEX); 423 setLastFreeClusterIndex(ROOT_CULSTER_INDEX); 424 } 425 426 setNextClusterIndex(nextFreeCluster , NULL_CLUSTER_INDEX); 428 setPreviousClusterIndex(NULL_CLUSTER_INDEX , nextFreeCluster); 429 return nextFreeCluster; 430 431 } 432 433 private boolean recoverFromFile(String fatFilePath){ 434 physicalFile = new File (fatFilePath); 435 if(physicalFile.exists() == false) 436 return false; 437 int needFileSize = ROOT_CULSTER_INDEX; 438 try { 439 randomAccessFile = new RandomAccessFile (physicalFile , "rw"); 440 441 lock = randomAccessFile.getChannel().tryLock(); 442 if(lock == null){ 443 log.fatal("Two applications are trying to read the same FAT, this is very bad (FAT not integrity safe), name="+this.physicalFile.getName()); 444 } 445 fatBuff = randomAccessFile.getChannel().map(FileChannel.MapMode.READ_WRITE ,0, needFileSize ); 446 447 fileInFatCount = loadFilesInFatCount(); 450 freeClustersCount = loadFreeClustersCount(); 452 assignedClusterCount = loadAssignedClusterCount(); 454 currentFileSize = loadCurrentFileSize(); 456 fatBuff = randomAccessFile.getChannel().map(FileChannel.MapMode.READ_WRITE ,0, currentFileSize ); 457 } catch (Exception e) { 458 459 if(log.isFatalEnabled()){ 460 log.fatal("could not load fat file "+fatFilePath,e); 461 return false; 462 } 464 } 465 466 return true; 467 } 468 469 private void createNewFatFile(String fatFilePath) throws IOException { 470 physicalFile = new File (fatFilePath); 472 randomAccessFile = new RandomAccessFile (physicalFile , "rw"); 473 int needFileSize = ROOT_CULSTER_INDEX+CLUSTER_SIZE; 474 lock = randomAccessFile.getChannel().tryLock(); 475 if(lock == null){ 476 log.fatal("Two applications are trying to read the same FAT, this is very bad (FAT not integrity safe), name="+this.physicalFile.getName()); 477 } 478 fatBuff = randomAccessFile.getChannel().map(FileChannel.MapMode.READ_WRITE ,0, needFileSize ); 479 currentFileSize = needFileSize; 480 saveCurrentFileSize(currentFileSize); 481 482 setPosition(FAT_TABLE_OFFSET); 484 for (int i = 0; i < MAX_FILES_IN_FAT; i++) { 485 fatBuff.putInt(NULL_CLUSTER_INDEX); 486 } 487 setClusterDataLength(ROOT_CULSTER_INDEX,(short)0); 489 setNextClusterIndex(ROOT_CULSTER_INDEX,NULL_CLUSTER_INDEX); 490 setPreviousClusterIndex( NULL_CLUSTER_INDEX, ROOT_CULSTER_INDEX); 491 allocateFreeClusters(INITIAL_CULSTER_CAPACITY); 493 494 495 } 496 497 501 private void allocateFreeClusters() throws IOException { 502 allocateFreeClusters(INCREMENT_CULSTER_VALUE); 503 504 } 505 510 private void allocateFreeClusters(int numberOfCluster) throws IOException { 511 int needFileSize = numberOfCluster*CLUSTER_SIZE; 512 fatBuff = randomAccessFile.getChannel().map(FileChannel.MapMode.READ_WRITE ,0, needFileSize+currentFileSize ); 513 int previous =ROOT_CULSTER_INDEX; 514 int clusterIndex = NULL_CLUSTER_INDEX; 515 for (int i = 0; i < numberOfCluster; i++) { 516 clusterIndex =getClusterIndex(assignedClusterCount+i); 517 linkClusters(previous,clusterIndex); 518 setClusterDataLength(clusterIndex , (short)0); 519 previous =clusterIndex; 520 } 521 setNextClusterIndex(clusterIndex,NULL_CLUSTER_INDEX); 522 setLastFreeClusterIndex(clusterIndex); 523 freeClustersCount = numberOfCluster; 524 saveFreeClustersCount(freeClustersCount); 525 currentFileSize =currentFileSize+needFileSize; 526 saveCurrentFileSize(currentFileSize); 527 } 528 529 534 private int getClusterIndex(int clusterCount){ 535 return ROOT_CULSTER_INDEX+(clusterCount*CLUSTER_SIZE); 536 } 537 538 private void setLastFreeClusterIndex(int lastClusterIndex){ 539 setPreviousClusterIndex(lastClusterIndex ,ROOT_CULSTER_INDEX); 541 } 542 543 private int getLastFreeClusterIndex(){ 544 return getPreviousClusterIndex(ROOT_CULSTER_INDEX); 546 } 547 548 554 private void setClusterDataLength(int clusterIndex ,short dataLength){ 555 setPosition(clusterIndex+CLUSTER_DATA_LENGTH_MD_OFFSET); 556 fatBuff.putShort(dataLength); 557 } 558 559 564 private short getClusterDataLength(int clusterIndex){ 565 setPosition(clusterIndex+CLUSTER_DATA_LENGTH_MD_OFFSET); 566 return fatBuff.getShort(); 567 } 568 569 572 private void linkClusters(int previousClusterIndex ,int nextClusterIndex){ 573 setNextClusterIndex( previousClusterIndex , nextClusterIndex); 574 setPreviousClusterIndex( previousClusterIndex , nextClusterIndex); 575 } 576 577 private void setNextClusterIndex(int previousClusterIndex ,int nextClusterIndex){ 578 setPosition(previousClusterIndex+CLUSTER_NEXT_MD_OFFSET); 579 fatBuff.putInt(nextClusterIndex); 580 } 581 582 private void setPreviousClusterIndex(int previousClusterIndex ,int nextClusterIndex){ 583 setPosition(nextClusterIndex+CLUSTER_PREVIOUS_MD_OFFSET); 584 fatBuff.putInt(previousClusterIndex); 585 } 586 587 private int getNextClusterIndex(int previousClusterIndex){ 588 setPosition(previousClusterIndex+CLUSTER_NEXT_MD_OFFSET); 589 return fatBuff.getInt(); 590 591 } 592 593 private int getPreviousClusterIndex(int nextClusterIndex){ 594 setPosition(nextClusterIndex+CLUSTER_PREVIOUS_MD_OFFSET); 595 return fatBuff.getInt(); 596 } 597 598 602 603 606 private void saveFilesInFatCount(int fileCount){ 607 setPosition(FAT_METADATA_OFFSET+METADATA_FILE_COUNT_OFFSET); 608 fatBuff.putInt(fileCount); 609 } 610 611 private int loadFilesInFatCount(){ 612 setPosition(FAT_METADATA_OFFSET+METADATA_FILE_COUNT_OFFSET); 613 return fatBuff.getInt(); 614 } 615 616 private void saveFreeClustersCount(int freeClustersCount){ 617 setPosition(FAT_METADATA_OFFSET+METADATA_FREE_CLUSTER_COUNT_OFFSET); 618 fatBuff.putInt(freeClustersCount); 619 } 620 621 private int loadFreeClustersCount(){ 622 setPosition(FAT_METADATA_OFFSET+METADATA_FREE_CLUSTER_COUNT_OFFSET); 623 return fatBuff.getInt(); 624 } 625 626 private void saveAssignedClusterCount(int assignedClusterCount){ 627 setPosition(FAT_METADATA_OFFSET+METADATA_ASSIGNED_CLUSTER_COUNT_OFFSET); 628 fatBuff.putInt(assignedClusterCount); 629 } 630 631 private int loadAssignedClusterCount(){ 632 setPosition(FAT_METADATA_OFFSET+METADATA_ASSIGNED_CLUSTER_COUNT_OFFSET); 633 return fatBuff.getInt(); 634 } 635 636 private void saveCurrentFileSize(int currentFileSize){ 637 setPosition(FAT_METADATA_OFFSET+METADATA_FAT_FILE_SIZE_OFFSET); 638 fatBuff.putInt(currentFileSize); 639 640 } 641 642 private int loadCurrentFileSize(){ 643 setPosition(FAT_METADATA_OFFSET+METADATA_FAT_FILE_SIZE_OFFSET); 644 return fatBuff.getInt(); 645 } 646 647 648 private final void setPosition(int pos){ 649 if(pos < 10 && pos % 4 != 0){ 650 LogFactory.getLog("MantaFatFile").debug("got to eror position"+pos); 651 } 652 if(fatBuff== null){ 653 throw new IllegalStateException ("FAT not created"); 654 } 655 fatBuff.position(pos); 656 } 657 658 659 662 public String toString(){ 663 StringBuffer str = new StringBuffer (); 664 str.append("\nMAX_FAT_FILE_SIZE ="+ MAX_FAT_FILE_SIZE ); 665 str.append("\nCLUSTER_SIZE ="+ CLUSTER_SIZE ); 666 str.append("\nMAX_FILES_IN_FAT ="+ MAX_FILES_IN_FAT); 667 str.append("\nBYTES_TO_INDEX_CLUSTER ="+ BYTES_TO_INDEX_CLUSTER); 668 str.append("\nFILE_TABLE_SIZE ="+ FILE_TABLE_SIZE ); 669 str.append("\nFAT_METADATA_SIZE ="+ FAT_METADATA_SIZE); 670 str.append("\nMAX_NUMBER_OF_CLUSTERS ="+ MAX_NUMBER_OF_CLUSTERS ); 671 str.append("\n fileInFatCount ="+fileInFatCount); 672 str.append("\n freeClustersCount ="+freeClustersCount); 673 str.append("\n assignedClusterCount ="+assignedClusterCount); 674 str.append("\n currentFileSize ="+currentFileSize); 675 676 return str.toString(); 677 } 678 679 680 681 684 public String getFatFilePath() { 685 return fatFilePath; 686 } 687 688 692 public synchronized void close() throws IOException { 693 lock.release(); 694 randomAccessFile.getChannel().force(true); 695 randomAccessFile.getChannel().close(); 696 fatBuff = null; 697 randomAccessFile.close(); 698 699 } 700 701 protected int getAssignedClusterCount() { 702 return assignedClusterCount; 703 } 704 protected int getCurrentFileSize() { 705 return currentFileSize; 706 } 707 protected int getFileInFatCount() { 708 return fileInFatCount; 709 } 710 protected int getFreeClustersCount() { 711 return freeClustersCount; 712 } 713 } 714 | Popular Tags |