RubyTokenizer


1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    * @authors: Zev Blut zb@ubit.com
4    */
5   package net.sourceforge.pmd.cpd;
6   
7   import java.util.List  ;
8   
9   public class RubyTokenizer implements Tokenizer {
10      private boolean downcaseString = true;
11  
12      public void tokenize(SourceCode tokens, Tokens tokenEntries) {
13          List   code = tokens.getCode();
14          for (int i = 0; i < code.size(); i++) {
15              String   currentLine = (String  ) code.get(i);
16              int loc = 0;
17              while (loc < currentLine.length()) {
18                  StringBuffer   token = new StringBuffer  ();
19                  loc = getTokenFromLine(currentLine, token, loc);
20                  if (token.length() > 0 && !isIgnorableString(token.toString())) {
21                      if (downcaseString) {
22                          token = new StringBuffer  (token.toString().toLowerCase());
23                      }
24                      tokenEntries.add(new TokenEntry(token.toString(),
25                              tokens.getFileName(),
26                              i + 1));
27                  }
28              }
29          }
30          tokenEntries.add(TokenEntry.getEOF());
31      }
32  
33      private int getTokenFromLine(String   line, StringBuffer   token, int loc) {
34          for (int j = loc; j < line.length(); j++) {
35              char tok = line.charAt(j);
36              if (!Character.isWhitespace(tok) && !ignoreCharacter(tok)) {
37                  if (isComment(tok)) {
38                      if (token.length() > 0) {
39                          return j;
40                      } else {
41                          return getCommentToken(line, token, loc);
42                      }
43                  } else if (isString(tok)) {
44                      if (token.length() > 0) {
45                          //if (loc == lin
46                          return j; // we need to now parse the string as a seperate token.
47                      } else {
48                          // we are at the start of a string
49                          return parseString(line, token, j, tok);
50                      }
51                  } else {
52                      token.append(tok);
53                  }
54              } else {
55                  if (token.length() > 0) {
56                      return j;
57                  }
58              }
59              loc = j;
60          }
61          return loc + 1;
62      }
63  
64      private int parseString(String   line, StringBuffer   token, int loc, char stringType) {
65          boolean escaped = false;
66          boolean done = false;
67          //System.out.println("Parsing String:" + stringType);
68          //System.out.println("Starting loc:" + loc);
69          // problem of strings that span multiple lines :-(
70          char tok = ' '; // this will be replaced.
71          while ((loc < line.length()) && !done) {
72              tok = line.charAt(loc);
73              if (escaped && tok == stringType) {
74                  //     System.out.println("Found an escaped string");
75                  escaped = false;
76              } else if (tok == stringType && (token.length() > 0)) {
77                  // we are done
78                  //   System.out.println("Found an end string");
79                  done = true;
80              } else if (tok == '\\') {
81                  // System.out.println("Found an escaped char");
82                  escaped = true;
83              } else {
84                  // System.out.println("Adding char:" + tok + ";loc:" + loc);
85                  escaped = false;
86              }
87              //System.out.println("Adding char to String:" + token.toString());
88              token.append(tok);
89              loc++;
90          }
91          return loc + 1;
92      }
93  
94      private boolean ignoreCharacter(char tok) {
95          boolean result = false;
96          switch (tok) {
97              case '{':
98              case '}':
99              case '(':
100             case ')':
101             case ';':
102             case ',':
103                 result = true;
104                 break;
105             default :
106                 result = false;
107         }
108         return result;
109     }
110 
111     private boolean isString(char tok) {
112         boolean result = false;
113         switch (tok) {
114             case '\'':
115             case '"':
116                 result = true;
117                 break;
118             default:
119                 result = false;
120         }
121         return result;
122     }
123 
124     private boolean isComment(char tok) {
125         return tok == '#';
126     }
127 
128     private int getCommentToken(String   line, StringBuffer   token, int loc) {
129         while (loc < line.length()) {
130             token.append(line.charAt(loc));
131             loc++;
132         }
133         return loc;
134     }
135 
136     private boolean isIgnorableString(String   token) {
137         return "do".equals(token) || "end".equals(token);
138     }
139 }
140
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags