KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > net > nutch > fs > LocalFileSystem


1 /* Copyright (c) 2004 The Nutch Organization. All rights reserved. */
2 /* Use subject to the conditions in http://www.nutch.org/LICENSE.txt. */
3
4 package net.nutch.fs;
5
6 import java.io.*;
7 import java.util.*;
8 import java.nio.channels.*;
9
10 import net.nutch.ndfs.NDFSFile;
11 import net.nutch.ndfs.NDFSFileInfo;
12 import net.nutch.io.UTF8;
13
14 /****************************************************************
15  * Implement the NutchFileSystem interface for the local disk.
16  * This is pretty easy. The interface exists so we can use either
17  * remote or local Files very easily.
18  *
19  * @author Mike Cafarella
20  *****************************************************************/
21 public class LocalFileSystem extends NutchFileSystem {
22     TreeMap sharedLockDataSet = new TreeMap();
23     TreeMap nonsharedLockDataSet = new TreeMap();
24     TreeMap lockObjSet = new TreeMap();
25     // by default use copy/delete instead of rename
26 boolean useCopyForRename = true;
27
28     /**
29      */
30     public LocalFileSystem() throws IOException {
31         super();
32         // if you find an OS which reliably supports non-POSIX
33 // rename(2) across filesystems / volumes, you can
34 // uncomment this.
35 // String os = System.getProperty("os.name");
36 // if (os.toLowerCase().indexOf("os-with-super-rename") != -1)
37 // useCopyForRename = false;
38 }
39
40     /*******************************************************
41      * For open()'s NFSInputStream
42      *******************************************************/
43     class LocalNFSFileInputStream extends NFSInputStream {
44         FileInputStream fis;
45
46         public LocalNFSFileInputStream(File f) throws IOException {
47           this.fis = new FileInputStream(f);
48         }
49
50         public void seek(long pos) throws IOException {
51           fis.getChannel().position(pos);
52         }
53
54         public long getPos() throws IOException {
55           return fis.getChannel().position();
56         }
57
58         /*
59          * Just forward to the fis
60          */
61         public int available() throws IOException { return fis.available(); }
62         public void close() throws IOException { fis.close(); }
63         public boolean markSupport() { return false; }
64         public int read() throws IOException { return fis.read(); }
65         public int read(byte[] b) throws IOException { return fis.read(b); }
66         public int read(byte[] b, int off, int len) throws IOException {
67             return fis.read(b, off, len);
68         }
69         public long skip(long n) throws IOException { return fis.skip(n); }
70     }
71     
72     /**
73      * Open the file at f
74      */
75     public NFSInputStream open(File f) throws IOException {
76         if (! f.exists()) {
77             throw new IOException("File does not exist");
78         }
79         return new LocalNFSFileInputStream(f);
80     }
81
82     /**
83      * Create the file at f.
84      */
85     public NFSOutputStream create(File f) throws IOException {
86         return create(f, false);
87     }
88
89     /*********************************************************
90      * For create()'s NFSOutputStream.
91      *********************************************************/
92     class LocalNFSFileOutputStream extends NFSOutputStream {
93       FileOutputStream fos;
94
95       public LocalNFSFileOutputStream(File f) throws IOException {
96         this.fos = new FileOutputStream(f);
97       }
98
99       public long getPos() throws IOException {
100         return fos.getChannel().position();
101       }
102
103       /*
104        * Just forward to the fos
105        */
106       public void close() throws IOException { fos.close(); }
107       public void flush() throws IOException { fos.flush(); }
108       public void write(byte[] b) throws IOException { fos.write(b); }
109       public void write(byte[] b, int off, int len) throws IOException {
110         fos.write(b, off, len);
111       }
112       public void write(int b) throws IOException { fos.write(b); }
113     }
114
115     /**
116      */
117     public NFSOutputStream create(File f, boolean overwrite) throws IOException {
118         if (f.exists() && ! overwrite) {
119             throw new IOException("File already exists");
120         }
121         File parent = f.getParentFile();
122         if (parent != null)
123           parent.mkdirs();
124
125         return new LocalNFSFileOutputStream(f);
126     }
127
128     /**
129      * Rename files/dirs
130      */
131     public boolean rename(File src, File dst) throws IOException {
132         if (useCopyForRename) {
133             FileUtil.copyContents(this, src, dst, true);
134             return fullyDelete(src);
135         } else return src.renameTo(dst);
136     }
137
138     /**
139      * Get rid of File f, whether a true file or dir.
140      */
141     public boolean delete(File f) throws IOException {
142         if (f.isFile()) {
143             return f.delete();
144         } else return fullyDelete(f);
145     }
146
147     /**
148      */
149     public boolean exists(File f) throws IOException {
150         return f.exists();
151     }
152
153     /**
154      */
155     public boolean isDirectory(File f) throws IOException {
156         return f.isDirectory();
157     }
158
159     /**
160      */
161     public long getLength(File f) throws IOException {
162         return f.length();
163     }
164
165     /**
166      */
167     public File[] listFiles(File f) throws IOException {
168         File[] files = f.listFiles();
169         if (files == null) return null;
170         // 20041022, xing, Watch out here:
171 // currently NDFSFile.java does not support those methods
172 // public boolean canRead()
173 // public boolean canWrite()
174 // public boolean createNewFile()
175 // public boolean delete()
176 // public void deleteOnExit()
177 // public boolean isHidden()
178 // so you can not rely on returned list for these operations.
179 NDFSFile[] nfiles = new NDFSFile[files.length];
180         for (int i = 0; i < files.length; i++) {
181             long len = files[i].length();
182             UTF8 name = new UTF8(files[i].toString());
183             NDFSFileInfo info = new NDFSFileInfo(name, len, len, files[i].isDirectory());
184             nfiles[i] = new NDFSFile(info);
185         }
186         return nfiles;
187     }
188
189     /**
190      */
191     public void mkdirs(File f) throws IOException {
192         f.mkdirs();
193     }
194
195     /**
196      * Obtain a filesystem lock at File f.
197      */
198     public synchronized void lock(File f, boolean shared) throws IOException {
199         f.createNewFile();
200
201         FileLock lockObj = null;
202         if (shared) {
203             FileInputStream lockData = new FileInputStream(f);
204             lockObj = lockData.getChannel().lock(0L, Long.MAX_VALUE, shared);
205             sharedLockDataSet.put(f, lockData);
206         } else {
207             FileOutputStream lockData = new FileOutputStream(f);
208             lockObj = lockData.getChannel().lock(0L, Long.MAX_VALUE, shared);
209             nonsharedLockDataSet.put(f, lockData);
210         }
211         lockObjSet.put(f, lockObj);
212     }
213
214     /**
215      * Release a held lock
216      */
217     public synchronized void release(File f) throws IOException {
218         FileLock lockObj = (FileLock) lockObjSet.get(f);
219         FileInputStream sharedLockData = (FileInputStream) sharedLockDataSet.get(f);
220         FileOutputStream nonsharedLockData = (FileOutputStream) nonsharedLockDataSet.get(f);
221
222         if (lockObj == null) {
223             throw new IOException("Given target not held as lock");
224         }
225         if (sharedLockData == null && nonsharedLockData == null) {
226             throw new IOException("Given target not held as lock");
227         }
228
229         lockObj.release();
230         lockObjSet.remove(f);
231         if (sharedLockData != null) {
232             sharedLockData.close();
233             sharedLockDataSet.remove(f);
234         } else {
235             nonsharedLockData.close();
236             nonsharedLockDataSet.remove(f);
237         }
238     }
239
240     /**
241      * In the case of the local filesystem, we can just rename the file.
242      */
243     public void moveFromLocalFile(File src, File dst) throws IOException {
244         if (! src.equals(dst)) {
245             if (useCopyForRename) {
246                 FileUtil.copyContents(this, src, dst, true);
247                 fullyDelete(src);
248             } else src.renameTo(dst);
249         }
250     }
251
252     /**
253      * Similar to moveFromLocalFile(), except the source is kept intact.
254      */
255     public void copyFromLocalFile(File src, File dst) throws IOException {
256         if (! src.equals(dst)) {
257             FileUtil.copyContents(this, src, dst, true);
258         }
259     }
260
261     /**
262      * We can't delete the src file in this case. Too bad.
263      */
264     public void copyToLocalFile(File src, File dst) throws IOException {
265         if (! src.equals(dst)) {
266             FileUtil.copyContents(this, src, dst, true);
267         }
268     }
269
270     /**
271      * We can write output directly to the final location
272      */
273     public File startLocalOutput(File nfsOutputFile, File tmpLocalFile) throws IOException {
274         return nfsOutputFile;
275     }
276
277     /**
278      * It's in the right place - nothing to do.
279      */
280     public void completeLocalOutput(File nfsWorkingFile, File tmpLocalFile) throws IOException {
281     }
282
283     /**
284      * We can read directly from the real local fs.
285      */
286     public File startLocalInput(File nfsInputFile, File tmpLocalFile) throws IOException {
287         return nfsInputFile;
288     }
289
290     /**
291      * We're done reading. Nothing to clean up.
292      */
293     public void completeLocalInput(File localFile) throws IOException {
294         // Ignore the file, it's at the right destination!
295 }
296
297     /**
298      * Create a temp file by just calling the Java method
299      */
300     public File createTempFile(String JavaDoc prefix, String JavaDoc suffix, File directory) throws IOException {
301         return new File(prefix + "-" + System.currentTimeMillis() + "-" + suffix);
302     }
303
304     /**
305      * Shut down the FS. Not necessary for regular filesystem.
306      */
307     public void close() throws IOException {
308     }
309
310     /**
311      */
312     public String JavaDoc toString() {
313         return "LocalFS";
314     }
315     
316     /**
317      * Implement our own version instead of using the one in FileUtil,
318      * to avoid infinite recursion.
319      * @param dir
320      * @return
321      * @throws IOException
322      */
323     private boolean fullyDelete(File dir) throws IOException {
324         File contents[] = dir.listFiles();
325         if (contents != null) {
326             for (int i = 0; i < contents.length; i++) {
327                 if (contents[i].isFile()) {
328                     if (! contents[i].delete()) {
329                         return false;
330                     }
331                 } else {
332                     if (! fullyDelete(contents[i])) {
333                         return false;
334                     }
335                 }
336             }
337         }
338         return dir.delete();
339     }
340 }
341
Popular Tags