|
1
19
20 package org.netbeans.lib.lexer.test.dump;
21
22 import java.io.File  ;
23 import java.io.FileNotFoundException ;
24 import java.io.FileReader ;
25 import java.io.FileWriter ;
26 import java.nio.CharBuffer ;
27 import java.util.ArrayList ;
28 import java.util.Arrays ;
29 import java.util.List ;
30 import org.netbeans.api.lexer.Language;
31 import org.netbeans.api.lexer.Token;
32 import org.netbeans.api.lexer.TokenHierarchy;
33 import org.netbeans.api.lexer.TokenHierarchy;
34 import org.netbeans.api.lexer.TokenId;
35 import org.netbeans.api.lexer.TokenSequence;
36 import org.netbeans.api.lexer.TokenUtilities;
37 import org.netbeans.junit.NbTestCase;
38 import org.netbeans.lib.editor.util.CharSequenceUtilities;
39 import org.netbeans.lib.lexer.batch.BatchTokenList;
40 import org.netbeans.lib.lexer.test.LexerTestUtilities;
41
42
47 public final class TokenDumpCheck {
48
49 public static void checkTokenDump(NbTestCase test, String relFilePath,
50 Language<? extends TokenId> language) throws Exception {
51 boolean origMaintainLAState = BatchTokenList.isMaintainLAState();
53 BatchTokenList.setMaintainLAState(true);
54 try {
55 File wholeInputFile = new File (test.getDataDir(), relFilePath);
56 if (!wholeInputFile.exists()) {
57 NbTestCase.fail("File " + wholeInputFile + " not found.");
58 }
59 String wholeInput = readFile(test, wholeInputFile);
60 TokenHierarchy<?> hi = TokenHierarchy.create(wholeInput, TokenDumpTokenId.language());
62 TokenSequence<TokenDumpTokenId> ts = hi.tokenSequence(TokenDumpTokenId.language());
63 boolean afterEOF = true; boolean newline = false;
65 int textStartIndex = 0;
66 List <String > inputs = new ArrayList <String >();
67 List <String > testNames = new ArrayList <String >();
68 StringBuilder inputBuffer = new StringBuilder (wholeInput.length() / 4);
70 String testName = "<Unnamed test>";
71 while (ts.moveNext()) {
72 Token<TokenDumpTokenId> token = ts.token();
73 switch (token.id()) {
74 case NEWLINE:
75 if (newline) { inputBuffer.append('\n');
77 }
78 if (!afterEOF) {
79 newline = true;
80 }
81 break;
82
83 case TEST_NAME:
84 testName = token.text().toString();
85 ts.moveNext(); newline = false;
87 break;
88
89 case EOF_VIRTUAL:
90 ts.moveNext(); newline = false;
93 inputs.add(inputBuffer.toString());
94 inputBuffer.setLength(0);
95 testNames.add(testName);
96 testName = "<Unnamed test>";
97 afterEOF = true;
98 break;
99
100 case TEXT:
101 if (newline) {
102 inputBuffer.append('\n');
103 newline = false;
104 }
105 inputBuffer.append(token.text());
106 afterEOF = false;
107 break;
108
109 default:
110 if (TokenDumpTokenId.isCharLiteral(token.id())) {
111 Character ch = (Character )token.getProperty(TokenDumpTokenId.UNICODE_CHAR_TOKEN_PROPERTY);
112 assert (ch != null);
113 inputBuffer.append(ch);
114 } else {
115 throw new IllegalStateException ("Unknown token id=" + token.id());
116 }
117 ts.moveNext(); newline = false;
119 afterEOF = false;
120
121 }
122 }
123 inputs.add(inputBuffer.toString());
124 testNames.add(testName);
125
126 File tokenDescInputFile = new File (test.getDataDir(), relFilePath + ".tokens.txt");
130 String tokenDescInputFilePath = tokenDescInputFile.getAbsolutePath();
131 boolean replaced = false;
132 if (tokenDescInputFilePath.indexOf("/build/test/") != -1) {
133 tokenDescInputFilePath = tokenDescInputFilePath.replace("/build/test/", "/test/");
134 replaced = true;
135 }
136 if (!replaced && tokenDescInputFilePath.indexOf("/test/work/sys/") != -1) {
137 tokenDescInputFilePath = tokenDescInputFilePath.replace("/test/work/sys/", "/test/unit/");
138 replaced = true;
139 }
140 if (!replaced) {
141 System.err.println("Warning: Attempt to use tokens dump file " +
142 "from sources instead of the generated test files failed.\n" +
143 "Patterns '/build/test/' or '/test/work/sys/' not found in " + tokenDescInputFilePath
144 );
145 }
146 tokenDescInputFile = new File (tokenDescInputFilePath);
147
148 String tokenDescInput = null;
149 if (tokenDescInputFile.exists()) {
150 tokenDescInput = readFile(test, tokenDescInputFile);
151 }
152 TokenDescCompare tdc = new TokenDescCompare(tokenDescInput, tokenDescInputFile);
153
154 StringBuilder tokenDesc = new StringBuilder (40);
156 for (int i = 0; i < inputs.size(); i++) {
157 String input = inputs.get(i);
158 testName = testNames.get(i);
159 tdc.setTestName(testName);
160 TokenHierarchy<?> langHi = TokenHierarchy.create(input, language);
161 TokenSequence<? extends TokenId> langTS = langHi.tokenSequence();
162 tdc.compareLine(testName, -1);
163 while (langTS.moveNext()) {
164 Token<? extends TokenId> token = langTS.token();
166 tokenDesc.append(token.id().name());
167 int spaceCount = 14 - token.id().name().length();
168 while (--spaceCount >= 0) {
169 tokenDesc.append(' ');
170 }
171 tokenDesc.append(" \"");
172 tokenDesc.append(TokenUtilities.debugText(token.text()));
173 tokenDesc.append('"');
174 int lookahead = LexerTestUtilities.lookahead(langTS);
175 if (lookahead > 0) {
176 tokenDesc.append(", la=");
177 tokenDesc.append(lookahead);
178 }
179 Object state = LexerTestUtilities.state(langTS);
180 if (state != null) {
181 tokenDesc.append(", st=");
182 tokenDesc.append(state);
183 }
184 tdc.compareLine(tokenDesc, langTS.index());
185 tokenDesc.setLength(0);
186 }
187 tdc.compareLine("----- EOF -----\n", -1);
188 }
189 tdc.finish();
191 } finally {
192 BatchTokenList.setMaintainLAState(origMaintainLAState);
193 }
194 }
195
196 private static String readFile(NbTestCase test, File f) throws Exception {
197 FileReader r = new FileReader (f);
198 int fileLen = (int)f.length();
199 CharBuffer cb = CharBuffer.allocate(fileLen);
200 r.read(cb);
201 cb.rewind();
202 return cb.toString();
203 }
204
205 private static final class TokenDescCompare implements CharSequence {
206
207 private String input;
208
209 private int inputIndex;
210
211 private int textLength;
212
213 private StringBuilder output;
214
215 private File outputFile;
216
217 private int[] lineBoundsIndexes = new int[2 * 3];
219 private String testName;
220
221 TokenDescCompare(String input, File outputFile) {
222 this.input = input;
223 this.outputFile = outputFile;
224 Arrays.fill(lineBoundsIndexes, -1);
225 if (input == null)
226 output = new StringBuilder (100);
227 }
228
229 public void compareLine(CharSequence text, int tokenIndex) {
230 if (input != null) {
231 textLength = text.length();
232 if (input.length() - inputIndex < textLength || !TokenUtilities.equals(text, this)) {
233 StringBuilder msg = new StringBuilder (100);
234 msg.append("\nDump file ");
235 msg.append(outputFile);
236 msg.append(":\n");
237 msg.append(testName);
238 msg.append(":\n");
239 if (tokenIndex >= 0) {
240 msg.append("Invalid token description in dump file (tokenIndex=");
241 msg.append(tokenIndex);
242 msg.append("):");
243 } else {
244 msg.append("Invalid text in dump file:");
245 }
246 msg.append("\n ");
247 msg.append(input.subSequence(inputIndex, findEOL(inputIndex)));
248 msg.append("\nExpected:\n ");
249 msg.append(text);
250 msg.append("\nPrevious context:\n");
251 for (int i = 0; i < lineBoundsIndexes.length; i += 2) {
252 int start = lineBoundsIndexes[i];
253 if (start != -1) {
254 msg.append(" ");
255 msg.append(input.subSequence(start, lineBoundsIndexes[i + 1]));
256 msg.append('\n');
257 }
258 }
259 NbTestCase.fail(msg.toString());
260
261 }
262 System.arraycopy(lineBoundsIndexes, 2, lineBoundsIndexes, 0, lineBoundsIndexes.length - 2);
263 lineBoundsIndexes[lineBoundsIndexes.length - 2] = inputIndex;
264 inputIndex += textLength;
265 lineBoundsIndexes[lineBoundsIndexes.length - 1] = inputIndex;
266 inputIndex = skipEOL(inputIndex);
267
268 } else {
269 output.append(text);
270 String ls = (String ) System.getProperty("line.separator"); output.append(ls);
272 }
273 }
274
275 public void finish() throws Exception {
276 if (input == null) {
277 if (!outputFile.createNewFile()) {
278 NbTestCase.fail("Cannot create file " + outputFile);
279 }
280 FileWriter fw = new FileWriter (outputFile);
281 try {
282 fw.write(output.toString());
283 } finally {
284 fw.close();
285 }
286 NbTestCase.fail("Created tokens dump file " + outputFile + "\nPlease re-run the test.");
287
288 } else {
289 if (inputIndex < input.length()) {
290 NbTestCase.fail("Some text left unread:" + input.substring(inputIndex));
291 }
292 }
293 }
294
295 public void setTestName(String testName) {
296 this.testName = testName;
297 }
298
299 public char charAt(int index) {
300 CharSequenceUtilities.checkIndexValid(index, length());
301 return input.charAt(inputIndex + index);
302 }
303
304 public int length() {
305 return textLength;
306 }
307
308 public CharSequence subSequence(int start, int end) {
309 CharSequenceUtilities.checkIndexesValid(this, start, end);
310 return input.substring(inputIndex + start, inputIndex + end);
311 }
312
313 private int findEOL(int start) {
314 while (start < input.length()) {
315 switch (input.charAt(start)) {
316 case '\r':
317 case '\n':
318 return start;
319 }
320 start++;
321 }
322 return start;
323 }
324
325 private int skipEOL(int index) {
326 if (index < input.length()) {
327 index++; if (input.charAt(index - 1) == '\r')
329 if (index < input.length() && input.charAt(index) == '\n')
330 index++; }
332 return index;
333 }
334
335 }
336
337 }
338 |
Popular Tags |
Java API By Example, From Geeks To Geeks.