KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > net > sourceforge > pmd > cpd > JavaTokenizer


1 /**
2  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3  */
4 package net.sourceforge.pmd.cpd;
5
6 import net.sourceforge.pmd.TargetJDK1_4;
7 import net.sourceforge.pmd.ast.JavaParserConstants;
8 import net.sourceforge.pmd.ast.JavaParserTokenManager;
9 import net.sourceforge.pmd.ast.Token;
10
11 import java.io.StringReader JavaDoc;
12 import java.util.Properties JavaDoc;
13
14 public class JavaTokenizer implements Tokenizer {
15
16     public static final String JavaDoc IGNORE_LITERALS = "ignore_literals";
17     public static final String JavaDoc IGNORE_IDENTIFIERS = "ignore_identifiers";
18
19     private boolean ignoreLiterals;
20     private boolean ignoreIdentifiers;
21
22     public void setProperties(Properties JavaDoc properties) {
23         ignoreLiterals = Boolean.valueOf(properties.getProperty(IGNORE_LITERALS, "false")).booleanValue();
24         ignoreIdentifiers = Boolean.valueOf(properties.getProperty(IGNORE_IDENTIFIERS, "false")).booleanValue();
25     }
26
27     public void tokenize(SourceCode tokens, Tokens tokenEntries) {
28         StringBuffer JavaDoc buffer = tokens.getCodeBuffer();
29
30         /*
31         I'm doing a sort of State pattern thing here where
32         this goes into "discarding" mode when it hits an import or package
33         keyword and goes back into "accumulate mode" when it hits a semicolon.
34         This could probably be turned into some objects.
35         */
36
37         // TODO - allow for JDK 1.5 selection
38 JavaParserTokenManager tokenMgr = new TargetJDK1_4().createJavaParserTokenManager(new StringReader JavaDoc(buffer.toString()));
39         Token currentToken = tokenMgr.getNextToken();
40         boolean inDiscardingState = false;
41         while (currentToken.image.length() > 0) {
42             if (currentToken.kind == JavaParserConstants.IMPORT || currentToken.kind == JavaParserConstants.PACKAGE) {
43                 inDiscardingState = true;
44                 currentToken = tokenMgr.getNextToken();
45                 continue;
46             }
47
48             if (inDiscardingState && currentToken.kind == JavaParserConstants.SEMICOLON) {
49                 inDiscardingState = false;
50             }
51
52             if (inDiscardingState) {
53                 currentToken = tokenMgr.getNextToken();
54                 continue;
55             }
56
57             if (currentToken.kind != JavaParserConstants.SEMICOLON) {
58                 String JavaDoc image = currentToken.image;
59                 if (ignoreLiterals && (currentToken.kind == JavaParserConstants.STRING_LITERAL || currentToken.kind == JavaParserConstants.CHARACTER_LITERAL
60                         || currentToken.kind == JavaParserConstants.DECIMAL_LITERAL || currentToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) {
61                     image = String.valueOf(currentToken.kind);
62                 }
63                 if (ignoreIdentifiers && currentToken.kind == JavaParserConstants.IDENTIFIER) {
64                     image = String.valueOf(currentToken.kind);
65                 }
66                 tokenEntries.add(new TokenEntry(image, tokens.getFileName(), currentToken.beginLine));
67             }
68
69             currentToken = tokenMgr.getNextToken();
70         }
71         tokenEntries.add(TokenEntry.getEOF());
72     }
73
74     public void setIgnoreLiterals(boolean ignore) {
75         this.ignoreLiterals = ignore;
76     }
77
78     public void setIgnoreIdentifiers(boolean ignore) {
79         this.ignoreIdentifiers = ignore;
80     }
81 }
82
Popular Tags