|
1 package net.javacoding.jspider.mod.plugin.statusbasedfilewriter;
2
3 import net.javacoding.jspider.api.event.EventVisitor;
4 import net.javacoding.jspider.api.event.JSpiderEvent;
5 import net.javacoding.jspider.api.event.engine.*;
6 import net.javacoding.jspider.api.event.folder.FolderDiscoveredEvent;
7 import net.javacoding.jspider.api.event.folder.FolderRelatedEvent;
8 import net.javacoding.jspider.api.event.resource.*;
9 import net.javacoding.jspider.api.event.site.*;
10 import net.javacoding.jspider.api.model.FetchTriedResource;
11 import net.javacoding.jspider.api.model.Resource;
12 import net.javacoding.jspider.core.logging.Log;
13 import net.javacoding.jspider.core.logging.LogFactory;
14 import net.javacoding.jspider.core.util.config.ConfigurationFactory;
15 import net.javacoding.jspider.spi.Plugin;
16
17 import java.io.*;
18 import java.util.*;
19
20
23 public class StatusBasedFileWriterPlugin implements Plugin, EventVisitor {
24
25 public static final String  MODULE_NAME = "Status based Filewriter JSpider plugin";
26 public static final String MODULE_VERSION = "v1.0";
27 public static final String MODULE_DESCRIPTION = "A JSpider plugin that writes a report file per HTTP status";
28 public static final String MODULE_VENDOR = "http://www.javacoding.net";
29
30 protected Log log;
31
32 protected HashMap fileWriters;
33
34 public StatusBasedFileWriterPlugin ( ) {
35 log = LogFactory.getLog ( StatusBasedFileWriterPlugin.class );
36 fileWriters = new HashMap ( );
37 log.info("initialized." );
38 }
39
40 public void initialize() {
41 }
42
43 public void shutdown() {
44 }
45
46 public String getName() {
47 return MODULE_NAME;
48 }
49
50 public String getVersion() {
51 return MODULE_VERSION;
52 }
53
54 public String getDescription() {
55 return MODULE_DESCRIPTION;
56 }
57
58 public String getVendor() {
59 return MODULE_VENDOR;
60 }
61
62 public void notify(JSpiderEvent event) {
63 event.accept(this);
64 }
65
66 public void visit(JSpiderEvent event) {
67 }
68
69 public void visit(EngineRelatedEvent event) {
70 }
71
72 public void visit(SpideringStartedEvent event) {
73 }
74
75 public void visit(SpideringStoppedEvent event) {
76 Collection printWriters = fileWriters.values();
77 Iterator it = printWriters.iterator();
78 while ( it.hasNext() ) {
79 PrintWriter pw = (PrintWriter)it.next();
80 pw.close();
81 }
82 }
83
84 public void visit(FolderRelatedEvent event) {
85 }
86
87 public void visit(FolderDiscoveredEvent event) {
88 }
89
90 public void visit(ResourceRelatedEvent event) {
91 }
92
93 public void visit(EMailAddressDiscoveredEvent event) {
94 }
95
96 public void visit(EMailAddressReferenceDiscoveredEvent event) {
97 }
98
99 public void visit(MalformedURLFoundEvent event) {
100 }
101
102 public void visit(MalformedBaseURLFoundEvent event) {
103 }
104
105 public void visit(ResourceDiscoveredEvent event) {
106 }
107
108 public void visit(ResourceFetchedEvent event) {
109 FetchTriedResource resource = event.getResource();
110 int state = resource.getHttpStatus();
111 writeInFile ( state, resource );
112 }
113
114 public void visit(ResourceFetchErrorEvent event) {
115 FetchTriedResource resource = event.getResource();
116 int state = resource.getHttpStatus();
117 writeInFileWithReferer ( state, resource );
118 }
119
120 public void visit(ResourceForbiddenEvent event) {
121 }
122
123 public void visit(ResourceParsedEvent event) {
124 }
125
126 public void visit(ResourceIgnoredForFetchingEvent event) {
127 }
128
129 public void visit(ResourceIgnoredForParsingEvent event) {
130 }
131
132 public void visit(ResourceReferenceDiscoveredEvent event) {
133 }
134
135 public void visit(SiteRelatedEvent event) {
136 }
137
138 public void visit(SiteDiscoveredEvent event) {
139 }
140
141 public void visit(RobotsTXTMissingEvent event) {
142 }
143
144 public void visit(RobotsTXTFetchedEvent event) {
145 }
146
147 public void visit(UserAgentObeyedEvent event) {
148 }
149
150 protected void writeInFile ( int state, Resource resource ) {
151 PrintWriter pw = getFileWriter(state);
152 pw.println(resource.getURL());
153 }
154
155 protected void writeInFileWithReferer ( int state, Resource resource ) {
156 PrintWriter pw = getFileWriter(state);
157 pw.println(resource.getURL());
158 pw.println(" REFERED BY:");
159 Resource[] referers = resource.getReferers();
160 for (int i = 0; i < referers.length; i++) {
161 Resource referer = referers[i];
162 pw.println(" " + referer.getURL() );
163 }
164 }
165
166 protected PrintWriter getFileWriter ( int state ) {
167 try {
168 Integer idObject = new Integer ( state );
169 PrintWriter retVal = (PrintWriter) fileWriters.get( ( idObject ));
170 if ( retVal == null ) {
171 log.info("creating file for status '" + state + "'" );
172 retVal = new PrintWriter ( new FileOutputStream (new File(ConfigurationFactory.getConfiguration().getDefaultOutputFolder(), state + ".out")));
173 log.debug("opened file for status '" + state + "'" );
174 fileWriters.put(idObject, retVal);
175 }
176 return retVal;
177 } catch (IOException e) {
178 log.error("i/o exception writing file for state " + state, e);
179 }
180 return null;
181 }
182
183 }
184 |
Popular Tags |
Java API By Example, From Geeks To Geeks.