1 2 3 4 package net.nutch.io; 5 6 import java.io.*; 7 import java.util.*; 8 import junit.framework.TestCase; 9 import java.util.logging.*; 10 11 import net.nutch.fs.*; 12 import net.nutch.util.*; 13 14 15 public class TestSetFile extends TestCase { 16 private static Logger LOG = SequenceFile.LOG; 17 private static String FILE = 18 System.getProperty("test.build.data",".") + "/test.set"; 19 20 public TestSetFile(String name) { super(name); } 21 22 public void testSetFile() throws Exception { 23 NutchFileSystem nfs = new LocalFileSystem(); 24 try { 25 RandomDatum[] data = generate(10000); 26 writeTest(nfs, data, FILE); 27 readTest(nfs, data, FILE); 28 } finally { 29 nfs.close(); 30 } 31 } 32 33 private static RandomDatum[] generate(int count) { 34 LOG.fine("generating " + count + " records in memory"); 35 RandomDatum[] data = new RandomDatum[count]; 36 RandomDatum.Generator generator = new RandomDatum.Generator(); 37 for (int i = 0; i < count; i++) { 38 generator.next(); 39 data[i] = generator.getValue(); 40 } 41 LOG.fine("sorting " + count + " records in memory"); 42 Arrays.sort(data); 43 return data; 44 } 45 46 private static void writeTest(NutchFileSystem nfs, RandomDatum[] data, String file) 47 throws IOException { 48 MapFile.delete(nfs, file); 49 LOG.fine("creating with " + data.length + " records"); 50 SetFile.Writer writer = new SetFile.Writer(nfs, file, RandomDatum.class); 51 for (int i = 0; i < data.length; i++) 52 writer.append(data[i]); 53 writer.close(); 54 } 55 56 private static void readTest(NutchFileSystem nfs, RandomDatum[] data, String file) 57 throws IOException { 58 RandomDatum v = new RandomDatum(); 59 LOG.fine("reading " + data.length + " records"); 60 SetFile.Reader reader = new SetFile.Reader(nfs, file); 61 for (int i = 0; i < data.length; i++) { 62 if (!reader.seek(data[i])) 63 throw new RuntimeException ("wrong value at " + i); 64 } 65 reader.close(); 66 LOG.fine("done reading " + data.length + " records"); 67 } 68 69 70 71 public static void main(String [] args) throws Exception { 72 int count = 1024 * 1024; 73 boolean create = true; 74 boolean check = true; 75 String file = FILE; 76 String usage = "Usage: TestSetFile (-local | -ndfs <namenode:port>) [-count N] [-nocreate] [-nocheck] file"; 77 78 if (args.length == 0) { 79 System.err.println(usage); 80 System.exit(-1); 81 } 82 83 int i = 0; 84 NutchFileSystem nfs = NutchFileSystem.parseArgs(args, i); 85 try { 86 for (; i < args.length; i++) { if (args[i] == null) { 88 continue; 89 } else if (args[i].equals("-count")) { 90 count = Integer.parseInt(args[++i]); 91 } else if (args[i].equals("-nocreate")) { 92 create = false; 93 } else if (args[i].equals("-nocheck")) { 94 check = false; 95 } else { 96 file = args[i]; 98 } 99 100 LOG.info("count = " + count); 101 LOG.info("create = " + create); 102 LOG.info("check = " + check); 103 LOG.info("file = " + file); 104 105 LOG.setLevel(Level.FINE); 106 107 RandomDatum[] data = generate(count); 108 109 if (create) { 110 writeTest(nfs, data, file); 111 } 112 113 if (check) { 114 readTest(nfs, data, file); 115 } 116 } 117 } finally { 118 nfs.close(); 119 } 120 } 121 } 122 | Popular Tags |