1 2 3 package net.nutch.fs; 4 5 import net.nutch.io.*; 6 import net.nutch.ipc.*; 7 import net.nutch.util.*; 8 import net.nutch.ndfs.*; 9 10 import java.io.*; 11 import java.net.*; 12 import java.util.*; 13 14 19 public class TestClient { 20 NutchFileSystem nfs; 21 22 24 public TestClient(NutchFileSystem nfs) { 25 this.nfs = nfs; 26 } 27 28 29 32 void copyFromLocal(File src, String dstf) throws IOException { 33 nfs.copyFromLocalFile(src, new File(dstf)); 34 } 35 36 39 void moveFromLocal(File src, String dstf) throws IOException { 40 nfs.moveFromLocalFile(src, new File(dstf)); 41 } 42 43 47 void copyToLocal(String srcf, File dst) throws IOException { 48 nfs.copyToLocalFile(new File(srcf), dst); 49 } 50 51 55 void moveToLocal(String srcf, File dst) throws IOException { 56 System.err.println("Option '-moveToLocal' is not implemented yet."); 57 } 58 59 62 public void ls(String src) throws IOException { 63 File items[] = nfs.listFiles(new File(src)); 64 if (items == null) { 65 System.out.println("Could not get listing for " + src); 66 } else { 67 System.out.println("Found " + items.length + " items"); 68 for (int i = 0; i < items.length; i++) { 69 File cur = items[i]; 70 System.out.println(cur.getPath() + "\t" + (cur.isDirectory() ? "<dir>" : ("" + cur.length()))); 71 } 72 } 73 } 74 75 77 public void du(String src) throws IOException { 78 File items[] = nfs.listFiles(new File(src)); 79 if (items == null) { 80 System.out.println("Could not get listing for " + src); 81 } else { 82 System.out.println("Found " + items.length + " items"); 83 for (int i = 0; i < items.length; i++) { 84 NDFSFile cur = (NDFSFile) items[i]; 85 System.out.println(cur.getPath() + "\t" + cur.getContentsLength()); 86 } 87 } 88 } 89 90 93 public void mkdir(String src) throws IOException { 94 File f = new File(src); 95 nfs.mkdirs(f); 96 } 97 98 101 public void rename(String srcf, String dstf) throws IOException { 102 if (nfs.rename(new File(srcf), new File(dstf))) { 103 System.out.println("Renamed " + srcf + " to " + dstf); 104 } else { 105 System.out.println("Rename failed"); 106 } 107 } 108 109 112 public void copy(String srcf, String dstf) throws IOException { 113 if (FileUtil.copyContents(nfs, new File(srcf), new File(dstf), true)) { 114 System.out.println("Copied " + srcf + " to " + dstf); 115 } else { 116 System.out.println("Copy failed"); 117 } 118 } 119 120 123 public void delete(String srcf) throws IOException { 124 if (nfs.delete(new File(srcf))) { 125 System.out.println("Deleted " + srcf); 126 } else { 127 System.out.println("Delete failed"); 128 } 129 } 130 131 134 String byteDesc(long len) { 135 double val = 0.0; 136 String ending = ""; 137 if (len < 1024 * 1024) { 138 val = (1.0 * len) / 1024; 139 ending = " k"; 140 } else if (len < 1024 * 1024 * 1024) { 141 val = (1.0 * len) / (1024 * 1024); 142 ending = " Mb"; 143 } else { 144 val = (1.0 * len) / (1024 * 1024 * 1024); 145 ending = " Gb"; 146 } 147 return limitDecimal(val, 2) + ending; 148 } 149 150 String limitDecimal(double d, int placesAfterDecimal) { 151 String strVal = Double.toString(d); 152 int decpt = strVal.indexOf("."); 153 if (decpt >= 0) { 154 strVal = strVal.substring(0, Math.min(strVal.length(), decpt + 1 + placesAfterDecimal)); 155 } 156 return strVal; 157 } 158 159 162 public void report() throws IOException { 163 if (nfs instanceof NDFSFileSystem) { 164 NDFSFileSystem ndfsfs = (NDFSFileSystem) nfs; 165 NDFSClient ndfs = ndfsfs.getClient(); 166 long total = ndfs.totalRawCapacity(); 167 long used = ndfs.totalRawUsed(); 168 DatanodeInfo info[] = ndfs.datanodeReport(); 169 170 long totalEffectiveBytes = 0; 171 File topItems[] = nfs.listFiles(new File("/")); 172 for (int i = 0; i < topItems.length; i++) { 173 NDFSFile cur = (NDFSFile) topItems[i]; 174 totalEffectiveBytes += cur.getContentsLength(); 175 } 176 177 System.out.println("Total raw bytes: " + total + " (" + byteDesc(total) + ")"); 178 System.out.println("Used raw bytes: " + used + " (" + byteDesc(used) + ")"); 179 System.out.println("% used: " + limitDecimal(((1.0 * used) / total) * 100, 2) + "%"); 180 System.out.println(); 181 System.out.println("Total effective bytes: " + totalEffectiveBytes + " (" + byteDesc(totalEffectiveBytes) + ")"); 182 System.out.println("Effective replication multiplier: " + (1.0 * used / totalEffectiveBytes)); 183 184 System.out.println("-------------------------------------------------"); 185 System.out.println("Datanodes available: " + info.length); 186 System.out.println(); 187 for (int i = 0; i < info.length; i++) { 188 System.out.println("Name: " + info[i].getName().toString()); 189 long c = info[i].getCapacity(); 190 long r = info[i].getRemaining(); 191 long u = c - r; 192 System.out.println("Total raw bytes: " + c + " (" + byteDesc(c) + ")"); 193 System.out.println("Used raw bytes: " + u + " (" + byteDesc(u) + ")"); 194 System.out.println("% used: " + limitDecimal(((1.0 * u) / c) * 100, 2) + "%"); 195 System.out.println("Last contact with namenode: " + new Date(info[i].lastUpdate())); 196 System.out.println(); 197 } 198 } 199 } 200 201 204 public static void main(String argv[]) throws IOException { 205 if (argv.length < 2) { 206 System.out.println("Usage: java NDFSClient (-local | -ndfs <namenode:port>) [-ls <path>] [-du <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm <src>] [-put <localsrc> <dst>] [-copyFromLocal <localsrc> <dst>] [-moveFromLocal <localsrc> <dst>] [-get <src> <localdst>] [-copyToLocal <src> <localdst>] [-moveToLocal <src> <localdst>]"); 207 return; 208 } 209 210 int i = 0; 211 NutchFileSystem nfs = NutchFileSystem.parseArgs(argv, i); 212 try { 213 TestClient tc = new TestClient(nfs); 214 215 String cmd = argv[i++]; 216 if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) { 217 tc.copyFromLocal(new File(argv[i++]), argv[i++]); 218 } else if ("-moveFromLocal".equals(cmd)) { 219 tc.moveFromLocal(new File(argv[i++]), argv[i++]); 220 } else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) { 221 tc.copyToLocal(argv[i++], new File(argv[i++])); 222 } else if ("-moveToLocal".equals(cmd)) { 223 tc.moveToLocal(argv[i++], new File(argv[i++])); 224 } else if ("-ls".equals(cmd)) { 225 tc.ls(argv[i++]); 226 } else if ("-mv".equals(cmd)) { 227 tc.rename(argv[i++], argv[i++]); 228 } else if ("-cp".equals(cmd)) { 229 tc.copy(argv[i++], argv[i++]); 230 } else if ("-rm".equals(cmd)) { 231 tc.delete(argv[i++]); 232 } else if ("-ls".equals(cmd)) { 233 tc.ls(argv[i++]); 234 } else if ("-du".equals(cmd)) { 235 tc.du(argv[i++]); 236 } else if ("-mkdir".equals(cmd)) { 237 tc.mkdir(argv[i++]); 238 } else if ("-report".equals(cmd)) { 239 tc.report(); 240 } 241 System.exit(0); 242 } finally { 243 nfs.close(); 244 } 245 } 246 } 247 | Popular Tags |