KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > net > nutch > db > EditSectionWriter


1 /* Copyright (c) 2003 The Nutch Organization. All rights reserved. */
2 /* Use subject to the conditions in http://www.nutch.org/LICENSE.txt. */
3 package net.nutch.db;
4
5 import java.io.*;
6 import java.util.*;
7
8 import net.nutch.io.*;
9 import net.nutch.fs.*;
10 import net.nutch.util.*;
11
12 /**********************************************
13  * EditSectionWriter writes a discrete portion of a WebDB.
14  * The WebDBWriter class may instantiate many EditSectionWriter
15  * objects to do its work (and always instantiates at
16  * least one).
17  *
18  * @author Mike Cafarella
19  ***********************************************/
20 public class EditSectionWriter {
21     private final int COMPLETION_VERSION = 0;
22     public final static String JavaDoc WRITE_METAINFO_PREFIX = "metainfo.";
23     public final static String JavaDoc EDITS_PREFIX = "edits.";
24
25     static int numSections;
26
27     /**
28      * Remove all the edits in this shareGroup from the indicated emitter.
29      */
30     /**
31     public static void removeEdits(NutchFile curSection, int emitter) throws IOException {
32         NutchFile edits = new NutchFile(curSection, EDITS_PREFIX + emitter);
33         NutchFile writeMetaInfo = new NutchFile(curSection, WRITE_METAINFO_PREFIX + emitter);
34         NutchFile writeCompletion = new NutchFile(curSection, WRITE_COMPLETION_PREFIX + emitter);
35
36         NutchFileSystem nfs = edits.getFS();
37         nfs.delete(edits);
38         nfs.delete(writeMetaInfo);
39         nfs.delete(writeCompletion);
40     }
41     **/
42
43     /**
44      * Remove all the edits in this section, from all emitters.
45      */
46     /**
47     public static void removeAllEdits(NutchFile curSection) throws IOException {
48         for (int i = 0; i < numSections; i++) {
49             removeEdits(curSection, i);
50         }
51     }
52     **/
53
54     
55     File editsDir, editsList;
56     File editsListFile;
57     int numEdits = 0;
58     boolean closed = false;
59     NutchFileSystem nfs;
60     SequenceFile.Writer seqWriter;
61
62     /**
63      * Make a EditSectionWriter for the appropriate file.
64      */
65     public EditSectionWriter(NutchFileSystem nfs, String JavaDoc label, int targetNum, int writerNum, Class JavaDoc keyClass, Class JavaDoc valClass) throws IOException {
66         this.nfs = nfs;
67
68         File allEditsDir = new File("editsection." + targetNum, "editsdir." + writerNum);
69         this.editsDir = new File(allEditsDir, label);
70         this.editsList = new File(editsDir, "editslist");
71         this.seqWriter = new SequenceFile.Writer(nfs, editsList.getPath(), keyClass, valClass);
72     }
73
74     /**
75      * Add a key/val pair
76      */
77     public synchronized void append(WritableComparable key, Writable val) throws IOException {
78         if (closed) {
79             throw new IOException("EditSectionWriter is closed");
80         }
81         seqWriter.append(key, val);
82         numEdits++;
83     }
84
85     /**
86      * Close down the EditSectionWriter. Afterwards, write down
87      * the completion file (including the number of edits inside).
88      */
89     public synchronized void close() throws IOException {
90         if (closed) {
91             throw new IOException("EditSectionWriter is already closed");
92         }
93
94         // Close down sequence writer
95 seqWriter.close();
96
97         // Separately write down the number of edits
98 File editsInfo = new File(editsDir, "editsinfo");
99         DataOutputStream out = new DataOutputStream(nfs.create(editsInfo));
100         try {
101             out.write(COMPLETION_VERSION);
102             out.writeInt(numEdits);
103         } finally {
104             out.close();
105         }
106     }
107 }
108
Popular Tags