|
1
4 package net.sourceforge.pmd.cpd;
5
6 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
7 import net.sourceforge.pmd.cpd.cppast.SimpleCharStream;
8 import net.sourceforge.pmd.cpd.cppast.Token;
9 import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
10
11 import java.io.StringReader  ;
12 import java.util.ArrayList ;
13 import java.util.List ;
14
15 public class CPPTokenizer implements Tokenizer {
16 protected String EOL = System.getProperty("line.separator", "\n");
17
18 private static SimpleCharStream charStream;
19
20 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
21 StringBuffer sb = sourceCode.getCodeBuffer();
22 try {
23 if (charStream == null) {
24 charStream = new SimpleCharStream(new StringReader (sb.toString()));
25 } else {
26 charStream.ReInit(new StringReader (sb.toString()));
27 }
28 CPPParserTokenManager.ReInit(charStream);
29 CPPParserTokenManager.setFileName(sourceCode.getFileName());
30 Token currToken = CPPParserTokenManager.getNextToken();
31 while (currToken.image.length() > 0) {
32 tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
33 currToken = CPPParserTokenManager.getNextToken();
34 }
35 tokenEntries.add(TokenEntry.getEOF());
36 System.out.println("Added " + sourceCode.getFileName());
37 } catch (TokenMgrError err) {
38 err.printStackTrace();
39 System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
40 List emptyCode = new ArrayList ();
41 emptyCode.add("");
42 tokenEntries.add(TokenEntry.getEOF());
43 }
44 }
45 }
46 |
Popular Tags |
Java API By Example, From Geeks To Geeks.