|
1
2
3
4 package net.nutch.io;
5
6 import java.io.*;
7 import java.util.*;
8 import junit.framework.TestCase;
9 import java.util.logging.*;
10
11 import net.nutch.fs.*;
12 import net.nutch.util.*;
13
14
15 public class TestArrayFile extends TestCase {
16 private static Logger LOG = SequenceFile.LOG;
17 private static String  FILE =
18 System.getProperty("test.build.data",".") + "/test.array";
19
20 public TestArrayFile(String name) {
21 super(name);
22 }
23
24 public void testArrayFile() throws Exception {
25 NutchFileSystem nfs = new LocalFileSystem();
26 RandomDatum[] data = generate(10000);
27 writeTest(nfs, data, FILE);
28 readTest(nfs, data, FILE);
29 }
30
31 public void testEmptyFile() throws Exception {
32 NutchFileSystem nfs = new LocalFileSystem();
33 writeTest(nfs, new RandomDatum[0], FILE);
34 ArrayFile.Reader reader = new ArrayFile.Reader(nfs, FILE);
35 assertNull(reader.get(0, new RandomDatum()));
36 reader.close();
37 }
38
39 private static RandomDatum[] generate(int count) {
40 LOG.fine("generating " + count + " records in memory");
41 RandomDatum[] data = new RandomDatum[count];
42 RandomDatum.Generator generator = new RandomDatum.Generator();
43 for (int i = 0; i < count; i++) {
44 generator.next();
45 data[i] = generator.getValue();
46 }
47 return data;
48 }
49
50 private static void writeTest(NutchFileSystem nfs, RandomDatum[] data, String file)
51 throws IOException {
52 MapFile.delete(nfs, file);
53 LOG.fine("creating with " + data.length + " records");
54 ArrayFile.Writer writer = new ArrayFile.Writer(nfs, file, RandomDatum.class);
55 writer.setIndexInterval(100);
56 for (int i = 0; i < data.length; i++)
57 writer.append(data[i]);
58 writer.close();
59 }
60
61 private static void readTest(NutchFileSystem nfs, RandomDatum[] data, String file)
62 throws IOException {
63 RandomDatum v = new RandomDatum();
64 LOG.fine("reading " + data.length + " records");
65 ArrayFile.Reader reader = new ArrayFile.Reader(nfs, file);
66 for (int i = 0; i < data.length; i++) { reader.get(i, v);
68 if (!v.equals(data[i])) {
69 throw new RuntimeException ("wrong value at " + i);
70 }
71 }
72 for (int i = data.length-1; i >= 0; i--) { reader.get(i, v);
74 if (!v.equals(data[i])) {
75 throw new RuntimeException ("wrong value at " + i);
76 }
77 }
78 reader.close();
79 LOG.fine("done reading " + data.length + " records");
80 }
81
82
83
84 public static void main(String [] args) throws Exception {
85 int count = 1024 * 1024;
86 boolean create = true;
87 boolean check = true;
88 String file = FILE;
89 String usage = "Usage: TestArrayFile (-local | -ndfs <namenode:port>) [-count N] [-nocreate] [-nocheck] file";
90
91 if (args.length == 0) {
92 System.err.println(usage);
93 System.exit(-1);
94 }
95
96 int i = 0;
97 NutchFileSystem nfs = NutchFileSystem.parseArgs(args, i);
98 try {
99 for (; i < args.length; i++) { if (args[i] == null) {
101 continue;
102 } else if (args[i].equals("-count")) {
103 count = Integer.parseInt(args[++i]);
104 } else if (args[i].equals("-nocreate")) {
105 create = false;
106 } else if (args[i].equals("-nocheck")) {
107 check = false;
108 } else {
109 file = args[i];
111 }
112 }
113
114 LOG.info("count = " + count);
115 LOG.info("create = " + create);
116 LOG.info("check = " + check);
117 LOG.info("file = " + file);
118
119 LOG.setLevel(Level.FINE);
120
121 RandomDatum[] data = generate(count);
122
123 if (create) {
124 writeTest(nfs, data, file);
125 }
126
127 if (check) {
128 readTest(nfs, data, file);
129 }
130 } finally {
131 nfs.close();
132 }
133 }
134 }
135 |
Popular Tags |
Java API By Example, From Geeks To Geeks.