|
1 package net.javacoding.jspider.core.util.html;
2
3 import junit.framework.TestCase;
4
5 import java.io.*;
6
7
10 public class RobotsTXTLineSetTest extends TestCase {
11
12 public static final String  TEST1 =
13 "user-agent: *" + "\n" +
14 "" + "\n" +
15 "disallow: /index.html" +"\n" +
16 "allow: /test" + "\n";
17
18 public static final String TEST2 =
19 "user-agent: testAgent" + "\n" +
20 "" + "\n" +
21 "allow: /index.html" +"\n" +
22 "allow: /test" + "\n" +
23 "" + "\n" +
24 "user-agent: test" + "\n" +
25 "" + "\n" +
26 "allow: /index.html" +"\n" +
27 "disallow: /test" + "\n" +
28 "" + "\n" +
29 "user-agent: *" + "\n" +
30 "" + "\n" +
31 "disallow: /index.html" +"\n" +
32 "disallow: /test" + "\n" +
33 "" + "\n";
34
35 public static final String TEST3 =
36 "user-agent: testAgent" + "\n" +
37 "" + "\n" +
38 "allow: /index.html" +"\n" +
39 "allow: /test" + "\n" +
40 "" + "\n" +
41 "user-agent: test" + "\n" +
42 "" + "\n" +
43 "allow: /index.html" +"\n" +
44 "disallow: /test" + "\n" +
45 "" + "\n" +
46 "user-agent: someAgent" + "\n" +
47 "" + "\n" +
48 "disallow: /index.html" +"\n" +
49 "disallow: /test" + "\n" +
50 "" + "\n";
51
52 public static final String TEST4 ="";
53
54 public static final String TEST5 =
55 "#user-agent: testAgent" + "\n" +
56 "" + "\n" +
57 "#allow: /index.html" +"\n" +
58 "#allow: /test" + "\n" +
59 "" + "\n" +
60 "#user-agent: test" + "\n" +
61 "" + "\n" +
62 "#allow: /index.html" +"\n" +
63 "#disallow: /test" + "\n" +
64 "" + "\n" +
65 "#user-agent: someAgent" + "\n" +
66 "" + "\n" +
67 "#disallow: /index.html" +"\n" +
68 "#disallow: /test" + "\n" +
69 "" + "\n";
70
71 public RobotsTXTLineSetTest ( ) {
72 super ( "RobotsTXTLineSetTest" );
73 }
74
75 protected BufferedReader getReader(String string) {
76 return new BufferedReader(new StringReader(string));
77 }
78
79 protected InputStream getInputStream(String string) {
80 return new ByteArrayInputStream(string.getBytes());
81 }
82
83 public void testSimpleRobotsTXT ( ) throws Exception {
84 BufferedReader br = getReader(TEST1);
85 RobotsTXTLineSet lineset = RobotsTXTLineSet.findLineSet(br, "testUserAgent");
86
87 assertEquals("lineset didn't contain exactly the one disallowing rule", 1, lineset.getLines().length);
88 assertEquals("expected to obey user agent '*'", "*", lineset.getUserAgent() );
89 }
90
91 public void testSimpleRobotsTXTViaInputStream ( ) throws Exception {
92 InputStream is = getInputStream(TEST1);
93 RobotsTXTLineSet lineset = RobotsTXTLineSet.findLineSet(is, "testUserAgent");
94
95 assertEquals("lineset didn't contain exactly the one disallowing rule", 1, lineset.getLines().length);
96 assertEquals("expected to obey user agent '*'", "*", lineset.getUserAgent() );
97 }
98
99 public void testUserAgentSelection1 ( ) throws Exception {
100 BufferedReader br = getReader(TEST2);
101 RobotsTXTLineSet lineset = RobotsTXTLineSet.findLineSet(br, "testUserAgent");
102
103 assertEquals("lineset didn't contain the right number of disallowing rules", 1, lineset.getLines().length);
104 assertEquals("expected to obey user agent 'test'", "test", lineset.getUserAgent() );
105 }
106
107 public void testUserAgentSelection2 ( ) throws Exception {
108 BufferedReader br = getReader(TEST2);
109 RobotsTXTLineSet lineset = RobotsTXTLineSet.findLineSet(br, "testAgent");
110
111 assertEquals("lineset didn't contain the right number of disallowing rules", 0, lineset.getLines().length);
112 assertEquals("expected to obey user agent 'testAgent'", "testAgent", lineset.getUserAgent() );
113 }
114
115 public void testUserAgentSelection3 ( ) throws Exception {
116 BufferedReader br = getReader(TEST2);
117 RobotsTXTLineSet lineset = RobotsTXTLineSet.findLineSet(br, "anotherAgent");
118
119 assertEquals("lineset didn't contain the right number of disallowing rules", 2, lineset.getLines().length);
120 assertEquals("expected to obey user agent '*'", "*", lineset.getUserAgent() );
121 }
122
123 public void testUserAgentSelectionEmptyString ( ) throws Exception {
124 BufferedReader br = getReader(TEST2);
125 RobotsTXTLineSet lineset = RobotsTXTLineSet.findLineSet(br, "");
126
127 assertEquals("lineset didn't contain the right number of disallowing rules", 2, lineset.getLines().length);
128 assertEquals("expected to obey user agent '*'", "*", lineset.getUserAgent() );
129 }
130
131 public void testUserAgentSelectionNullString ( ) throws Exception {
132 BufferedReader br = getReader(TEST2);
133 RobotsTXTLineSet lineset = RobotsTXTLineSet.findLineSet(br, null);
134
135 assertEquals("lineset didn't contain the right number of disallowing rules", 2, lineset.getLines().length);
136 assertEquals("expected to obey user agent '*'", "*", lineset.getUserAgent() );
137 }
138
139 public void testUserAgentNotListed ( ) throws Exception {
140 BufferedReader br = getReader(TEST3);
141 RobotsTXTLineSet lineset = RobotsTXTLineSet.findLineSet(br, "unlistedUA");
142
143 assertNull("lineset wasn't null for non-listed agent", lineset);
144 }
145
146 public void testEmptyRobotsTXT ( ) throws Exception {
147 BufferedReader br = getReader(TEST4);
148 RobotsTXTLineSet lineset = RobotsTXTLineSet.findLineSet(br, "testUserAgent");
149
150 assertNull ("lineset wasn't null for empty robots.txt", lineset );
151 }
152
153 public void testCommentEmptyRobotsTXT ( ) throws Exception {
154 BufferedReader br = getReader(TEST5);
155 RobotsTXTLineSet lineset = RobotsTXTLineSet.findLineSet(br, "testUserAgent");
156
157 assertNull ("lineset wasn't null for empty robots.txt", lineset );
158 }
159
160 }
161 |
Popular Tags |
Java API By Example, From Geeks To Geeks.