KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > net > nutch > io > TestSetFile


1 /* Copyright (c) 2003 The Nutch Organization. All rights reserved. */
2 /* Use subject to the conditions in http://www.nutch.org/LICENSE.txt. */
3
4 package net.nutch.io;
5
6 import java.io.*;
7 import java.util.*;
8 import junit.framework.TestCase;
9 import java.util.logging.*;
10
11 import net.nutch.fs.*;
12 import net.nutch.util.*;
13
14 /** Support for flat files of binary key/value pairs. */
15 public class TestSetFile extends TestCase {
16   private static Logger LOG = SequenceFile.LOG;
17   private static String JavaDoc FILE =
18     System.getProperty("test.build.data",".") + "/test.set";
19
20   public TestSetFile(String JavaDoc name) { super(name); }
21
22   public void testSetFile() throws Exception JavaDoc {
23     NutchFileSystem nfs = new LocalFileSystem();
24     try {
25         RandomDatum[] data = generate(10000);
26         writeTest(nfs, data, FILE);
27         readTest(nfs, data, FILE);
28     } finally {
29         nfs.close();
30     }
31   }
32
33   private static RandomDatum[] generate(int count) {
34     LOG.fine("generating " + count + " records in memory");
35     RandomDatum[] data = new RandomDatum[count];
36     RandomDatum.Generator generator = new RandomDatum.Generator();
37     for (int i = 0; i < count; i++) {
38       generator.next();
39       data[i] = generator.getValue();
40     }
41     LOG.fine("sorting " + count + " records in memory");
42     Arrays.sort(data);
43     return data;
44   }
45
46   private static void writeTest(NutchFileSystem nfs, RandomDatum[] data, String JavaDoc file)
47     throws IOException {
48     MapFile.delete(nfs, file);
49     LOG.fine("creating with " + data.length + " records");
50     SetFile.Writer writer = new SetFile.Writer(nfs, file, RandomDatum.class);
51     for (int i = 0; i < data.length; i++)
52       writer.append(data[i]);
53     writer.close();
54   }
55
56   private static void readTest(NutchFileSystem nfs, RandomDatum[] data, String JavaDoc file)
57     throws IOException {
58     RandomDatum v = new RandomDatum();
59     LOG.fine("reading " + data.length + " records");
60     SetFile.Reader reader = new SetFile.Reader(nfs, file);
61     for (int i = 0; i < data.length; i++) {
62       if (!reader.seek(data[i]))
63         throw new RuntimeException JavaDoc("wrong value at " + i);
64     }
65     reader.close();
66     LOG.fine("done reading " + data.length + " records");
67   }
68
69
70   /** For debugging and testing. */
71   public static void main(String JavaDoc[] args) throws Exception JavaDoc {
72     int count = 1024 * 1024;
73     boolean create = true;
74     boolean check = true;
75     String JavaDoc file = FILE;
76     String JavaDoc usage = "Usage: TestSetFile (-local | -ndfs <namenode:port>) [-count N] [-nocreate] [-nocheck] file";
77       
78     if (args.length == 0) {
79       System.err.println(usage);
80       System.exit(-1);
81     }
82       
83     int i = 0;
84     NutchFileSystem nfs = NutchFileSystem.parseArgs(args, i);
85     try {
86       for (; i < args.length; i++) { // parse command line
87
if (args[i] == null) {
88           continue;
89         } else if (args[i].equals("-count")) {
90           count = Integer.parseInt(args[++i]);
91         } else if (args[i].equals("-nocreate")) {
92           create = false;
93         } else if (args[i].equals("-nocheck")) {
94           check = false;
95         } else {
96           // file is required parameter
97
file = args[i];
98         }
99
100         LOG.info("count = " + count);
101         LOG.info("create = " + create);
102         LOG.info("check = " + check);
103         LOG.info("file = " + file);
104
105         LOG.setLevel(Level.FINE);
106
107         RandomDatum[] data = generate(count);
108
109         if (create) {
110           writeTest(nfs, data, file);
111         }
112
113         if (check) {
114           readTest(nfs, data, file);
115         }
116       }
117     } finally {
118       nfs.close();
119     }
120   }
121 }
122
Popular Tags