Tokenizer


1   /*******************************************************************************
2    * Copyright (c) 2003, 2005 IBM Corporation and others.
3    * All rights reserved. This program and the accompanying materials
4    * are made available under the terms of the Eclipse Public License v1.0
5    * which accompanies this distribution, and is available at
6    * http://www.eclipse.org/legal/epl-v10.html
7    *
8    * Contributors:
9    *     IBM Corporation - initial API and implementation
10   *******************************************************************************/
11  
12  package org.eclipse.osgi.framework.internal.core;
13  
14  /**
15   * Simple tokenizer class. Used to parse data.
16   */
17  public class Tokenizer {
18      protected char value[];
19      protected int max;
20      protected int cursor;
21  
22      public Tokenizer(String   value) {
23          this.value = value.toCharArray();
24          max = this.value.length;
25          cursor = 0;
26      }
27  
28      private void skipWhiteSpace() {
29          char[] val = value;
30          int cur = cursor;
31  
32          for (; cur < max; cur++) {
33              char c = val[cur];
34              if ((c == ' ') || (c == '\t') || (c == '\n') || (c == '\r')) {
35                  continue;
36              }
37              break;
38          }
39          cursor = cur;
40      }
41  
42      public String   getToken(String   terminals) {
43          skipWhiteSpace();
44          char[] val = value;
45          int cur = cursor;
46  
47          int begin = cur;
48          for (; cur < max; cur++) {
49              char c = val[cur];
50              if ((terminals.indexOf(c) != -1)) {
51                  break;
52              }
53          }
54          cursor = cur;
55          int count = cur - begin;
56          if (count > 0) {
57              skipWhiteSpace();
58              while (count > 0 && (val[begin + count - 1] == ' ' || val[begin + count - 1] == '\t'))
59                  count--;
60              return (new String  (val, begin, count));
61          }
62          return (null);
63      }
64  
65      public String   getString(String   terminals) {
66          skipWhiteSpace();
67          char[] val = value;
68          int cur = cursor;
69  
70          if (cur < max) {
71              if (val[cur] == '\"') /* if a quoted string */
72              {
73                  StringBuffer   sb = new StringBuffer  ();
74                  cur++; /* skip quote */
75                  char c = '\0';
76                  int begin = cur;
77                  for (; cur < max; cur++) {
78                      c = val[cur];
79                      // this is an escaped char
80                      if (c == '\\') {
81                          cur++; // skip the escape char
82                          if (cur == max)
83                              break;
84                          c = val[cur]; // include the escaped char
85                      } else if (c == '\"') {
86                          break;
87                      }
88                      sb.append(c);
89                  }
90                  int count = cur - begin;
91                  if (c == '\"') {
92                      cur++;
93                  }
94                  cursor = cur;
95                  if (count > 0) {
96                      skipWhiteSpace();
97                      return sb.toString();
98                  }
99              } else /* not a quoted string; same as token */
100             {
101                 return getToken(terminals);
102             }
103         }
104         return (null);
105     }
106 
107     public char getChar() {
108         int cur = cursor;
109         if (cur < max) {
110             cursor = cur + 1;
111             return (value[cur]);
112         }
113         return ('\0'); /* end of value */
114     }
115 
116     public boolean hasMoreTokens() {
117         if (cursor < max) {
118             return true;
119         }
120         return false;
121     }
122 }
123
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags