Tokenizer


1   /*******************************************************************************
2    * Copyright (c) 2003, 2004 IBM Corporation and others.
3    * All rights reserved. This program and the accompanying materials
4    * are made available under the terms of the Common Public License v1.0
5    * which accompanies this distribution, and is available at
6    * http://www.eclipse.org/legal/cpl-v10.html
7    * 
8    * Contributors:
9    *     IBM Corporation - initial API and implementation
10   *******************************************************************************/
11  
12  package org.eclipse.osgi.framework.util;
13  
14  /**
15   * Simple tokenizer class. Used to parse data.
16   */
17  public class Tokenizer {
18      protected char value[];
19      protected int max;
20      protected int cursor;
21  
22      public Tokenizer(String   value) {
23          this.value = value.toCharArray();
24          max = this.value.length;
25          cursor = 0;
26      }
27  
28      private void skipWhiteSpace() {
29          char[] val = value;
30          int cur = cursor;
31  
32          for (; cur < max; cur++) {
33              char c = val[cur];
34              if ((c == ' ') || (c == '\t') || (c == '\n') || (c == '\r')) {
35                  continue;
36              }
37              break;
38          }
39          cursor = cur;
40      }
41  
42      public String   getToken(String   terminals) {
43          skipWhiteSpace();
44          char[] val = value;
45          int cur = cursor;
46  
47          int begin = cur;
48          for (; cur < max; cur++) {
49              char c = val[cur];
50              if ((terminals.indexOf(c) != -1)) {
51                  break;
52              }
53          }
54          cursor = cur;
55          int count = cur - begin;
56          if (count > 0) {
57              skipWhiteSpace();
58              while (count > 0 && (val[begin + count - 1] == ' ' || val[begin + count - 1] == '\t'))
59                  count--;
60              return (new String  (val, begin, count));
61          }
62          return (null);
63      }
64  
65      public String   getString(String   terminals) {
66          skipWhiteSpace();
67          char[] val = value;
68          int cur = cursor;
69  
70          if (cur < max) {
71              if (val[cur] == '\"') /* if a quoted string */
72              {
73                  cur++; /* skip quote */
74                  char c = '\0';
75                  int begin = cur;
76                  for (; cur < max; cur++) {
77                      c = val[cur];
78                      if (c == '\"') {
79                          break;
80                      }
81                  }
82                  int count = cur - begin;
83                  if (c == '\"') {
84                      cur++;
85                  }
86                  cursor = cur;
87                  if (count > 0) {
88                      skipWhiteSpace();
89                      return (new String  (val, begin, count));
90                  }
91              } else /* not a quoted string; same as token */
92              {
93                  int begin = cur;
94                  for (; cur < max; cur++) {
95                      char c = val[cur];
96                      if (c == '\"') {
97                          // but there could be a quoted string in the middle of the string
98                          cur = cur + skipQuotedString(val, cur);
99                      } else if ((terminals.indexOf(c) != -1)) {
100                         break;
101                     }
102                 }
103                 cursor = cur;
104                 int count = cur - begin;
105                 if (count > 0) {
106                     skipWhiteSpace();
107                     while (count > 0 && (val[begin + count - 1] == ' ' || val[begin + count - 1] == '\t'))
108                         count--;
109                     return (new String  (val, begin, count));
110                 }
111             }
112         }
113         return (null);
114     }
115 
116     private int skipQuotedString(char[] val, int cur) {
117         cur++; /* skip quote */
118         char c = '\0';
119         int begin = cur;
120         for (; cur < max; cur++) {
121             c = val[cur];
122             if (c == '\"') {
123                 break;
124             }
125         }
126         int count = cur - begin;
127         if (c == '\"') {
128             cur++;
129         }
130         cursor = cur;
131         if (count > 0) {
132             skipWhiteSpace();
133         }
134         return count;
135     }
136 
137     public char getChar() {
138         int cur = cursor;
139         if (cur < max) {
140             cursor = cur + 1;
141             return (value[cur]);
142         }
143         return ('\0'); /* end of value */
144     }
145 
146     public boolean hasMoreTokens() {
147         if (cursor < max) {
148             return true;
149         }
150         return false;
151     }
152 }
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags