KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > net > sourceforge > pmd > cpd > AnyTokenizer


1 /**
2  * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3  */

4 package net.sourceforge.pmd.cpd;
5
6 import java.io.BufferedReader JavaDoc;
7 import java.io.CharArrayReader JavaDoc;
8 import java.util.NoSuchElementException JavaDoc;
9 import java.util.StringTokenizer JavaDoc;
10
11 /**
12  * This class does a best-guess try-anything tokenization.
13  *
14  * @author jheintz
15  */

16 public class AnyTokenizer implements Tokenizer {
17     public static final String JavaDoc TOKENS = " \t!#$%^&*(){}-=+<>/\\`~;:";
18
19     public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
20         StringBuffer JavaDoc sb = sourceCode.getCodeBuffer();
21         BufferedReader JavaDoc reader = new BufferedReader JavaDoc(new CharArrayReader JavaDoc(sb.toString().toCharArray()));
22         try {
23             int lineNumber = 1;
24             String JavaDoc line = reader.readLine();
25             while (line != null) {
26                 StringTokenizer JavaDoc tokenizer = new StringTokenizer JavaDoc(line, TOKENS, true);
27                 try {
28                     String JavaDoc token = tokenizer.nextToken();
29                     while (token != null) {
30                         if (!token.equals(" ") && !token.equals("\t")) {
31                             tokenEntries.add(new TokenEntry(token, sourceCode.getFileName(), lineNumber));
32                         }
33                         token = tokenizer.nextToken();
34                     }
35                 } catch (NoSuchElementException JavaDoc ex) {
36                     // done with tokens
37
}
38                 // advance iteration variables
39
line = reader.readLine();
40                 lineNumber++;
41             }
42         } catch (Exception JavaDoc ex) {
43             ex.printStackTrace();
44         } finally {
45             try {
46                 reader.close();
47             } catch (Exception JavaDoc ex) {
48             }
49             tokenEntries.add(TokenEntry.getEOF());
50         }
51     }
52 }
53
Popular Tags