|
1 package net.javacoding.jspider.mod.plugin.diskwriter;
2
3 import net.javacoding.jspider.api.event.JSpiderEvent;
4 import net.javacoding.jspider.api.event.resource.ResourceFetchedEvent;
5 import net.javacoding.jspider.core.logging.Log;
6 import net.javacoding.jspider.core.logging.LogFactory;
7 import net.javacoding.jspider.core.util.config.*;
8 import net.javacoding.jspider.spi.Plugin;
9
10 import java.io.*;
11 import java.net.URL  ;
12 import java.util.StringTokenizer ;
13
14
15
20 public class DiskWriterPlugin implements Plugin {
21
22 public static final String MODULE_NAME = "DiskWriter JSpider plugin";
23 public static final String MODULE_VERSION = "v1.0";
24 public static final String MODULE_DESCRIPTION = "A JSpider plugin that allows the mirroring of web resources";
25 public static final String MODULE_VENDOR = "http://www.javacoding.net";
26
27 public static final String OUTPUT_ABSOLUTE = "output.absolute";
28 public static final String OUTPUT_FOLDER = "output.folder";
29
30 public static final String DEFAULT_OUTPUT_FOLDER = ".";
31
32 protected Log log;
33
34 protected File outputFolder;
35 protected File baseFolder;
36
37 protected PrintWriter pw;
38
39 public DiskWriterPlugin(PropertySet config) {
40 log = LogFactory.getLog(DiskWriterPlugin.class);
41 JSpiderConfiguration jspiderConfig = ConfigurationFactory.getConfiguration();
42 File defaultOutputFolder = jspiderConfig.getDefaultOutputFolder();
43 if (config.getBoolean(OUTPUT_ABSOLUTE, false)) {
44 outputFolder = new File(config.getString(OUTPUT_FOLDER, DEFAULT_OUTPUT_FOLDER));
45 baseFolder = new File("/");
46 } else {
47 outputFolder = new File(defaultOutputFolder, config.getString(OUTPUT_FOLDER, DEFAULT_OUTPUT_FOLDER));
48 baseFolder = new File(".");
49 }
50 log.info("Writing to output folder: " + outputFolder);
51 }
52
53 public void initialize() {
54 }
55
56 public void shutdown() {
57 }
58
59 public String getName() {
60 return MODULE_NAME;
61 }
62
63 public String getVersion() {
64 return MODULE_VERSION;
65 }
66
67 public String getDescription() {
68 return MODULE_DESCRIPTION;
69 }
70
71 public String getVendor() {
72 return MODULE_VENDOR;
73 }
74
75 public void notify(JSpiderEvent event) {
76 if (event instanceof ResourceFetchedEvent) {
77 ResourceFetchedEvent rfe = (ResourceFetchedEvent) event;
78 URL url = rfe.getURL();
79 String path = url.getHost() + url.getPath();
80 if (includesFile(url)) {
81 path = url.getHost() + url.getFile();
82 } else {
83 path = url.getHost() + url.getPath() + "/index.html";
84 }
85 File outputFile = new File(outputFolder, path);
86 ensureFolders(outputFile);
87 log.debug("Writing " + outputFile);
88 writeFile(outputFile, rfe.getResource().getInputStream());
89 log.info("Wrote " + outputFile);
90 }
91 }
92
93 protected void writeFile(File outputFile, InputStream is) {
94 try {
95 FileOutputStream fos = new FileOutputStream(outputFile);
96 int read = is.read();
97 while (read != -1) {
98 fos.write(read);
99 read = is.read();
100 }
101 fos.close();
102 } catch (IOException e) {
103 log.error("i/o error writing file", e);
104 }
105 }
106
107 protected void ensureFolders( File folder) {
108 String path = folder.getPath();
109 StringTokenizer st = new StringTokenizer (path, File.separator);
110 ensureFolder(baseFolder, st);
111 }
112
113 protected void ensureFolder(File current, StringTokenizer st) {
114 String thisToken = st.nextToken();
116 if (st.hasMoreTokens()) {
117 File thisFolder = new File(current, thisToken);
118 if (!thisFolder.exists()) {
119 thisFolder.mkdir();
120 }
121 ensureFolder(thisFolder, st);
122 }
123 }
124
125 protected boolean includesFile(URL url) {
126 String urlString = url.getPath();
127 return (urlString.lastIndexOf(".") > urlString.lastIndexOf('/'));
128 }
129
130 }
131 |
Popular Tags |
Java API By Example, From Geeks To Geeks.