KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > net > nutch > fs > TestClient


1 /* Copyright (c) 2004 The Nutch Organization. All rights reserved. */
2 /* Use subject to the conditions in http://www.nutch.org/LICENSE.txt. */
3 package net.nutch.fs;
4
5 import net.nutch.io.*;
6 import net.nutch.ipc.*;
7 import net.nutch.util.*;
8 import net.nutch.ndfs.*;
9
10 import java.io.*;
11 import java.net.*;
12 import java.util.*;
13
14 /**************************************************
15  * This class provides some NDFS administrative access.
16  *
17  * @author Mike Cafarella
18  **************************************************/

19 public class TestClient {
20     NutchFileSystem nfs;
21
22     /**
23      */

24     public TestClient(NutchFileSystem nfs) {
25         this.nfs = nfs;
26     }
27
28
29     /**
30      * Add a local file to the indicated name in NDFS. src is kept.
31      */

32     void copyFromLocal(File src, String JavaDoc dstf) throws IOException {
33         nfs.copyFromLocalFile(src, new File(dstf));
34     }
35
36     /**
37      * Add a local file to the indicated name in NDFS. src is removed.
38      */

39     void moveFromLocal(File src, String JavaDoc dstf) throws IOException {
40         nfs.moveFromLocalFile(src, new File(dstf));
41     }
42
43     /**
44      * Obtain the indicated NDFS file and copy to the local name.
45      * srcf is kept.
46      */

47     void copyToLocal(String JavaDoc srcf, File dst) throws IOException {
48         nfs.copyToLocalFile(new File(srcf), dst);
49     }
50
51     /**
52      * Obtain the indicated NDFS file and copy to the local name.
53      * srcf is removed.
54      */

55     void moveToLocal(String JavaDoc srcf, File dst) throws IOException {
56         System.err.println("Option '-moveToLocal' is not implemented yet.");
57     }
58
59     /**
60      * Get a listing of all files in NDFS at the indicated name
61      */

62     public void ls(String JavaDoc src) throws IOException {
63         File items[] = nfs.listFiles(new File(src));
64         if (items == null) {
65             System.out.println("Could not get listing for " + src);
66         } else {
67             System.out.println("Found " + items.length + " items");
68             for (int i = 0; i < items.length; i++) {
69                 File cur = items[i];
70                 System.out.println(cur.getPath() + "\t" + (cur.isDirectory() ? "<dir>" : ("" + cur.length())));
71             }
72         }
73     }
74
75     /**
76      */

77     public void du(String JavaDoc src) throws IOException {
78         File items[] = nfs.listFiles(new File(src));
79         if (items == null) {
80             System.out.println("Could not get listing for " + src);
81         } else {
82             System.out.println("Found " + items.length + " items");
83             for (int i = 0; i < items.length; i++) {
84                 NDFSFile cur = (NDFSFile) items[i];
85                 System.out.println(cur.getPath() + "\t" + cur.getContentsLength());
86             }
87         }
88     }
89
90     /**
91      * Create the given dir
92      */

93     public void mkdir(String JavaDoc src) throws IOException {
94         File f = new File(src);
95         nfs.mkdirs(f);
96     }
97     
98     /**
99      * Rename an NDFS file
100      */

101     public void rename(String JavaDoc srcf, String JavaDoc dstf) throws IOException {
102         if (nfs.rename(new File(srcf), new File(dstf))) {
103             System.out.println("Renamed " + srcf + " to " + dstf);
104         } else {
105             System.out.println("Rename failed");
106         }
107     }
108
109     /**
110      * Copy an NDFS file
111      */

112     public void copy(String JavaDoc srcf, String JavaDoc dstf) throws IOException {
113         if (FileUtil.copyContents(nfs, new File(srcf), new File(dstf), true)) {
114             System.out.println("Copied " + srcf + " to " + dstf);
115         } else {
116             System.out.println("Copy failed");
117         }
118     }
119
120     /**
121      * Delete an NDFS file
122      */

123     public void delete(String JavaDoc srcf) throws IOException {
124         if (nfs.delete(new File(srcf))) {
125             System.out.println("Deleted " + srcf);
126         } else {
127             System.out.println("Delete failed");
128         }
129     }
130
131     /**
132      * Return an abbreviated English-language desc of the byte length
133      */

134     String JavaDoc byteDesc(long len) {
135         double val = 0.0;
136         String JavaDoc ending = "";
137         if (len < 1024 * 1024) {
138             val = (1.0 * len) / 1024;
139             ending = " k";
140         } else if (len < 1024 * 1024 * 1024) {
141             val = (1.0 * len) / (1024 * 1024);
142             ending = " Mb";
143         } else {
144             val = (1.0 * len) / (1024 * 1024 * 1024);
145             ending = " Gb";
146         }
147         return limitDecimal(val, 2) + ending;
148     }
149
150     String JavaDoc limitDecimal(double d, int placesAfterDecimal) {
151         String JavaDoc strVal = Double.toString(d);
152         int decpt = strVal.indexOf(".");
153         if (decpt >= 0) {
154             strVal = strVal.substring(0, Math.min(strVal.length(), decpt + 1 + placesAfterDecimal));
155         }
156         return strVal;
157     }
158
159     /**
160      * Gives a report on how the NutchFileSystem is doing
161      */

162     public void report() throws IOException {
163         if (nfs instanceof NDFSFileSystem) {
164             NDFSFileSystem ndfsfs = (NDFSFileSystem) nfs;
165             NDFSClient ndfs = ndfsfs.getClient();
166             long total = ndfs.totalRawCapacity();
167             long used = ndfs.totalRawUsed();
168             DatanodeInfo info[] = ndfs.datanodeReport();
169
170             long totalEffectiveBytes = 0;
171             File topItems[] = nfs.listFiles(new File("/"));
172             for (int i = 0; i < topItems.length; i++) {
173                 NDFSFile cur = (NDFSFile) topItems[i];
174                 totalEffectiveBytes += cur.getContentsLength();
175             }
176
177             System.out.println("Total raw bytes: " + total + " (" + byteDesc(total) + ")");
178             System.out.println("Used raw bytes: " + used + " (" + byteDesc(used) + ")");
179             System.out.println("% used: " + limitDecimal(((1.0 * used) / total) * 100, 2) + "%");
180             System.out.println();
181             System.out.println("Total effective bytes: " + totalEffectiveBytes + " (" + byteDesc(totalEffectiveBytes) + ")");
182             System.out.println("Effective replication multiplier: " + (1.0 * used / totalEffectiveBytes));
183
184             System.out.println("-------------------------------------------------");
185             System.out.println("Datanodes available: " + info.length);
186             System.out.println();
187             for (int i = 0; i < info.length; i++) {
188                 System.out.println("Name: " + info[i].getName().toString());
189                 long c = info[i].getCapacity();
190                 long r = info[i].getRemaining();
191                 long u = c - r;
192                 System.out.println("Total raw bytes: " + c + " (" + byteDesc(c) + ")");
193                 System.out.println("Used raw bytes: " + u + " (" + byteDesc(u) + ")");
194                 System.out.println("% used: " + limitDecimal(((1.0 * u) / c) * 100, 2) + "%");
195                 System.out.println("Last contact with namenode: " + new Date(info[i].lastUpdate()));
196                 System.out.println();
197             }
198         }
199     }
200
201     /**
202      * main() has some simple utility methods
203      */

204     public static void main(String JavaDoc argv[]) throws IOException {
205         if (argv.length < 2) {
206             System.out.println("Usage: java NDFSClient (-local | -ndfs <namenode:port>) [-ls <path>] [-du <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm <src>] [-put <localsrc> <dst>] [-copyFromLocal <localsrc> <dst>] [-moveFromLocal <localsrc> <dst>] [-get <src> <localdst>] [-copyToLocal <src> <localdst>] [-moveToLocal <src> <localdst>]");
207             return;
208         }
209
210         int i = 0;
211         NutchFileSystem nfs = NutchFileSystem.parseArgs(argv, i);
212         try {
213             TestClient tc = new TestClient(nfs);
214
215             String JavaDoc cmd = argv[i++];
216             if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) {
217                 tc.copyFromLocal(new File(argv[i++]), argv[i++]);
218             } else if ("-moveFromLocal".equals(cmd)) {
219                 tc.moveFromLocal(new File(argv[i++]), argv[i++]);
220             } else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
221                 tc.copyToLocal(argv[i++], new File(argv[i++]));
222             } else if ("-moveToLocal".equals(cmd)) {
223                 tc.moveToLocal(argv[i++], new File(argv[i++]));
224             } else if ("-ls".equals(cmd)) {
225                 tc.ls(argv[i++]);
226             } else if ("-mv".equals(cmd)) {
227                 tc.rename(argv[i++], argv[i++]);
228             } else if ("-cp".equals(cmd)) {
229                 tc.copy(argv[i++], argv[i++]);
230             } else if ("-rm".equals(cmd)) {
231                 tc.delete(argv[i++]);
232             } else if ("-ls".equals(cmd)) {
233                 tc.ls(argv[i++]);
234             } else if ("-du".equals(cmd)) {
235                 tc.du(argv[i++]);
236             } else if ("-mkdir".equals(cmd)) {
237                 tc.mkdir(argv[i++]);
238             } else if ("-report".equals(cmd)) {
239                 tc.report();
240             }
241             System.exit(0);
242         } finally {
243             nfs.close();
244         }
245     }
246 }
247
Popular Tags