KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > net > nutch > fs > NutchFileSystem


1 /* Copyright (c) 2004 The Nutch Organization. All rights reserved. */
2 /* Use subject to the conditions in http://www.nutch.org/LICENSE.txt. */
3 package net.nutch.fs;
4
5 import java.io.*;
6 import java.net.*;
7 import java.util.*;
8 import java.util.logging.*;
9
10 import net.nutch.ndfs.*;
11 import net.nutch.util.*;
12
13 /****************************************************************
14  * NutchFileSystem is an interface for a fairly simple
15  * distributed file system. A Nutch installation might consist
16  * of multiple machines, which should swap files transparently.
17  * This interface allows other Nutch systems to find and place
18  * files into the distributed Nutch-controlled file world.
19  *
20  * The standard job of NutchFileSystem is to take the location-
21  * independent NutchFile objects, and resolve them using local
22  * knowledge and local instances of ShareGroup.
23  *
24  * @author Mike Cafarella
25  *****************************************************************/
26 public abstract class NutchFileSystem {
27     public static final Logger LOG = LogFormatter.getLogger("net.nutch.util.NutchFileSystem");
28
29     /**
30      * Parse the cmd-line args, starting at i. Remove consumed args
31      * from array. We expect param in the form:
32      * '-local | -ndfs <namenode:port>'
33      */
34     public static NutchFileSystem parseArgs(String JavaDoc argv[], int i) throws IOException {
35         /**
36         if (argv.length - i < 1) {
37             throw new IOException("Must indicate filesystem type for NDFS");
38         }
39         */
40         int orig = i;
41         NutchFileSystem nfs = null;
42         String JavaDoc cmd = argv[i];
43         if ("-ndfs".equals(cmd)) {
44             i++;
45             InetSocketAddress addr = NDFS.createSocketAddr(argv[i++]);
46             nfs = new NDFSFileSystem(addr);
47         } else if ("-local".equals(cmd)) {
48             i++;
49             nfs = new LocalFileSystem();
50         } else {
51             LOG.info("No NutchFileSystem indicated, so defaulting to local fs.");
52             nfs = new LocalFileSystem();
53         }
54         System.arraycopy(argv, i, argv, orig, argv.length - i);
55         for (int j = argv.length - i; j < argv.length; j++) {
56             argv[j] = null;
57         }
58         return nfs;
59     }
60
61     ///////////////////////////////////////////////////////////////
62 // NutchFileSystem
63 ///////////////////////////////////////////////////////////////
64 /**
65      */
66     public NutchFileSystem() {
67     }
68
69     /**
70      * Opens an InputStream for the indicated File, whether local
71      * or via NDFS.
72      */
73     public abstract NFSInputStream open(File f) throws IOException;
74
75     /**
76      * Opens an OutputStream at the indicated File, whether local
77      * or via NDFS.
78      */
79     public abstract NFSOutputStream create(File f) throws IOException;
80     public abstract NFSOutputStream create(File f, boolean overwrite) throws IOException;
81
82     /**
83      * Creates the given File as a brand-new zero-length file. If
84      * create fails, or if it already existed, return false.
85      */
86     public boolean createNewFile(File f) throws IOException {
87         if (exists(f)) {
88             return false;
89         } else {
90             OutputStream out = create(f);
91             try {
92             } finally {
93                 out.close();
94             }
95             return true;
96         }
97     }
98
99     /**
100      * Renames File src to File dst. Can take place on local fs
101      * or remote NDFS.
102      */
103     public abstract boolean rename(File src, File dst) throws IOException;
104
105     /**
106      * Deletes File
107      */
108     public abstract boolean delete(File f) throws IOException;
109
110     /**
111      * Check if exists
112      */
113     public abstract boolean exists(File f) throws IOException;
114
115     /**
116      */
117     public abstract boolean isDirectory(File f) throws IOException;
118
119     /**
120      */
121     public boolean isFile(File f) throws IOException {
122         if (exists(f) && ! isDirectory(f)) {
123             return true;
124         } else {
125             return false;
126         }
127     }
128     
129     /**
130      */
131     public abstract long getLength(File f) throws IOException;
132
133     /**
134      */
135     public abstract File[] listFiles(File f) throws IOException;
136
137     public File[] listFiles(File f, FileFilter filter) throws IOException {
138         Vector results = new Vector();
139         File listing[] = listFiles(f);
140         for (int i = 0; i < listing.length; i++) {
141             if (filter.accept(listing[i])) {
142                 results.add(listing[i]);
143             }
144         }
145         return (File[]) results.toArray(new File[results.size()]);
146     }
147
148     /**
149      * Make the given file and all non-existent parents into
150      * directories.
151      */
152     public abstract void mkdirs(File f) throws IOException;
153
154     /**
155      * Obtain a lock on the given File
156      */
157     public abstract void lock(File f, boolean shared) throws IOException;
158
159     /**
160      * Release the lock
161      */
162     public abstract void release(File f) throws IOException;
163
164     /**
165      * The src file is on the local disk. Add it to NFS at
166      * the given dst name and the source is kept intact afterwards
167      */
168     // not implemneted yet
169 public abstract void copyFromLocalFile(File src, File dst) throws IOException;
170
171     /**
172      * The src file is on the local disk. Add it to NFS at
173      * the given dst name, removing the source afterwards.
174      */
175     public abstract void moveFromLocalFile(File src, File dst) throws IOException;
176
177     /**
178      * The src file is under NFS2, and the dst is on the local disk.
179      * Copy it from NFS control to the local dst name.
180      */
181     public abstract void copyToLocalFile(File src, File dst) throws IOException;
182
183     /**
184      * the same as copyToLocalFile(File src, File dst), except that
185      * the source is removed afterward.
186      */
187     // not implemented yet
188 //public abstract void moveToLocalFile(File src, File dst) throws IOException;
189
190     /**
191      * Returns a local File that the user can write output to. The caller
192      * provides both the eventual NFS target name and the local working
193      * file. If the NFS is local, we write directly into the target. If
194      * the NFS is remote, we write into the tmp local area.
195      */
196     public abstract File startLocalOutput(File nfsOutputFile, File tmpLocalFile) throws IOException;
197
198     /**
199      * Called when we're all done writing to the target. A local NFS will
200      * do nothing, because we've written to exactly the right place. A remote
201      * NFS will copy the contents of tmpLocalFile to the correct target at
202      * nfsOutputFile.
203      */
204     public abstract void completeLocalOutput(File nfsOutputFile, File tmpLocalFile) throws IOException;
205
206     /**
207      * Returns a local File that the user can read from. The caller
208      * provides both the eventual NFS target name and the local working
209      * file. If the NFS is local, we read directly from the source. If
210      * the NFS is remote, we write data into the tmp local area.
211      */
212     public abstract File startLocalInput(File nfsInputFile, File tmpLocalFile) throws IOException;
213
214     /**
215      * Called when we're all done writing to the target. A local NFS will
216      * do nothing, because we've written to exactly the right place. A remote
217      * NFS will copy the contents of tmpLocalFile to the correct target at
218      * nfsOutputFile.
219      */
220     public abstract void completeLocalInput(File localFile) throws IOException;
221
222     /**
223      * Create an empty File in the given directory (or /tmp, if directory is
224      * null) using the given prefix and suffix to guide name generation.
225      */
226     public abstract File createTempFile(String JavaDoc prefix, String JavaDoc suffix, File directory) throws IOException;
227
228     /**
229      * No more filesystem operations are needed. Will
230      * release any held locks.
231      */
232     public abstract void close() throws IOException;
233 }
234
Popular Tags