|
1
6 package tests.jfun.parsec;
7
8 import junit.framework.TestCase;
9 import junit.framework.TestSuite;
10 import junit.framework.Test;
11 import jfun.parsec.*;
12 import jfun.parsec.tokens.Tokens;
13
14
19 public class LexerTestCase extends TestCase {
20 private static Tok[] scan(final String  src, final Parser p){
21 return (Tok[])Parsers.runParser(src, p,
22 new DefaultPositionMap(src, 1, 1), "TestLexer");
23 }
24 public void test1(){
25 try{
26 final Tok[] toks = scan("1+2", lexer1());
27 assertEquals(3, toks.length);
28 assertEquals(Tokens.int_literal(1), toks[0].getToken());
29 assertEquals("+", toks[1].getToken().toString());
30 assertEquals(Tokens.int_literal(2), toks[2].getToken());
31
38 }
39 catch(ParserException e){
40 fail(e.getMessage());
41 }
42 }
43 public void test2(){
44 try{
45 final Tok[] toks = scan("", lexer1());
46 assertEquals(0, toks.length);
47 }
48 catch(ParserException e){
49 fail(e.getMessage());
50 }
51 }
52 public void test3(){
53 try{
54 final Tok[] toks = scan(" ", lexer1());
55 assertEquals(0, toks.length);
56 }
57 catch(ParserException e){
58 fail(e.getMessage());
59 }
60 }
61 public void test4(){
62 try{
63 final Tok[] toks = scan(" /*hello*/", lexer1());
64 assertEquals(0, toks.length);
65 }
66 catch(ParserException e){
67 fail(e.getMessage());
68 }
69 }
70 public void test5(){
71 try{
72 final Tok[] toks = scan(" //this is a test\n ", lexer1());
73 assertEquals(0, toks.length);
74 }
75 catch(ParserException e){
76 fail(e.getMessage());
77 }
78 }
79 private static final class Result{
80 final Object tok;
81 final int lno;
82 final int cno;
83 Result(final int lno, final int cno, final Object t){
84 this.lno = lno;
85 this.cno = cno;
86 this.tok = t;
87 }
88 }
89 private static Result result(final int lno, final int cno, final Object tok){
90 return new Result(lno, cno, tok);
91 }
92 private static void assertTokens(final Tok[] vals, final Result[] expected){
93 assertEquals(expected.length, vals.length);
94 for(int i=0; i<expected.length; i++){
95 final Tok pt = vals[i];
96 final Result r = expected[i];
97 assertEquals(r.tok, pt.getToken());
98 }
99 }
100 public void test6(){
101 try{
102 final Tok[] toks = scan("1 *(\r\n1 +20)", lexer1());
103 assertTokens(toks, new Result[]{
104 result(1,1,Tokens.int_literal(1)),
105 result(1,3,words.getToken("*")),
106 result(1,4,words.getToken("(")),
107 result(2,1,Tokens.int_literal(1)),
108 result(2,3,words.getToken("+")),
109 result(2,4,Tokens.int_literal(20)),
110 result(2,6,words.getToken(")"))
111 });
112 }
113 catch(ParserException e){
114 fail(e.getMessage());
115 }
116 }
117 public void test7(){
118 try{
119 final Tok[] toks = scan("1 *(\r\n1 +0x2a)", lexer1());
120 assertTokens(toks, new Result[]{
121 result(1,1,Tokens.int_literal(1)),
122 result(1,3,words.getToken("*")),
123 result(1,4,words.getToken("(")),
124 result(2,1,Tokens.int_literal(1)),
125 result(2,3,words.getToken("+")),
126 result(2,4,Tokens.int_literal(42)),
127 result(2,8,words.getToken(")"))
128 });
129 }
130 catch(ParserException e){
131 fail(e.getMessage());
132 }
133 }
134 public void test8(){
135 testBad(lexer1(), "1 *(\r\n1 +0x)");
136 }
137 public void test9(){
138 testBad(lexer1(), "1 *(\r\n1 +0x10/*hello no end)");
139 }
140 public void test10(){
141 System.out.println(testBad(lexer1(), "/*h"));
142 }
143 private static final Words words = Lexers.getOperators(new String []{
144 "+", "-", "*", "/", "(", ")"
145 });
146 private static Parser lexer1(){
147 final Parser ts1 = Parsers.sum(new Parser[]{Lexers.lexLong(), words.getLexer()});
151 final Parser delim = Scanners.javaDelimiter();
152 final Parser ts = Lexers.lexeme(delim, ts1);
153 return ts.followedBy(Parsers.eof());
154 }
155 private static Parser lexer2(){
156 final Parser delimiter = Scanners.isChar(' ').seq(Scanners.isChar('@').some(3));
157 return Lexers.lexeme(delimiter.optional(), Lexers.integer()).followedBy(Parsers.eof());
158 }
159 public void testPound1(){
160 testGood(lexer2(), "123", 1);
161 }
162 public void testPound2(){
163 testGood(lexer2(), "123 2", 2);
164 }
165 public void testPound3(){
166 testGood(lexer2(), "123 2 @@@", 2);
167 }
168 public void testPound4(){
169 testGood(lexer2(), "123 @@2 @@@", 2);
170 }
171 public void testPound5(){
172 System.out.println(testBad(lexer2(), "123 @@@@2 @@@"));
173 }
174 public void test11(){
175 testBad(lexer1(), "1 *(\r\n1 +09)\n");
176 }
177 private static String bigCommentStr(){
178 final StringBuffer buf = new StringBuffer ();
179 for(int i=0; i<100; i++){
180 buf.append(i).append(" \t\r\n");
181 buf.append("//");
182 for(int j=0;j<40;j++){
183 buf.append(j).append(" /");
184 }
185 buf.append("\n* /*");
186 for(int j=0;j<1000;j++){
187 buf.append(j).append(" /**");
188 }
189 buf.append("*/");
190 }
191 return buf.toString();
192 }
193 private static String smallCommentStr(){
194 final StringBuffer buf = new StringBuffer ();
195 for(int i=0; i<10000; i++){
196 buf.append(i).append(" \t\r\n");
197 buf.append("//");
198 for(int j=0;j<10;j++){
199 buf.append(j).append(" /");
200 }
201 buf.append("\n* /*");
202 for(int j=0;j<1;j++){
203 buf.append(j).append(" /**");
204 }
205 buf.append("*/");
206 }
207 return buf.toString();
208 }
209 private static String noCommentStr(){
210 final StringBuffer buf = new StringBuffer ();
211 for(int i=0; i<10000; i++){
212 buf.append(i).append(" \t\r\n");
213 for(int j=0;j<10;j++){
214 buf.append(" ");
215 }
216 buf.append("\n* ");
217 for(int j=0;j<1;j++){
218 buf.append(" ");
219 }
220 }
221 return buf.toString();
222 }
223 private static void runSpeedTest(final Parser lx, final String src, final int tn){
224 final long begin = System.currentTimeMillis();
225
226 testGood(lx, src, tn);
227 System.out.println("took " + (System.currentTimeMillis()-begin) + "ms");
228 }
229 public void testSpeed1(){
230 runSpeedTest(lexer1(), bigCommentStr(), 200);
231 }
232 public void testSpeed2(){
233 runSpeedTest(lexer1(), smallCommentStr(), 20000);
234 }
235 public void testSpeed3(){
236 runSpeedTest(lexer1(), noCommentStr(), 20000);
237 }
238 private static void testGood(final Parser lx, final String s, final int tn){
239 try{
240 final Tok[] toks = scan(s, lx);
241 assertEquals(tn, toks.length);
242 }
243 catch(ParserException e){
244 fail(e.getMessage());
245 }
246 }
247 private static String testBad(final Parser ts, final String s){
248 try{
249 scan(s, ts);
250 fail("should have failed!");
251 return null;
252 }
253 catch(ParserException e){
254 return e.getMessage();
255 }
256 }
257 public static Test suite(){
258 return new TestSuite(LexerTestCase.class);
259 }
260 public static void main(String [] args){
261 junit.textui.TestRunner.run(suite());
262 }
263
264 }
265 |
Popular Tags |
Java API By Example, From Geeks To Geeks.