|
1
2 package org.jahia.services.search.analyzer;
3
4 import java.io.*;
5
6
14 public class StandardTokenizer extends org.apache.lucene.analysis.Tokenizer implements StandardTokenizerConstants {
15
16
17 public StandardTokenizer(Reader reader) {
18 this(new FastCharStream(reader));
19 this.input = reader;
20 }
21
22
26 final public org.apache.lucene.analysis.Token next() throws IOException {
27 Token token = null;
28 try {
29 switch ( (jj_ntk == -1) ? jj_ntk() : jj_ntk) {
30 case ALPHANUM:
31 token = jj_consume_token(ALPHANUM);
32 break;
33 case APOSTROPHE:
34 token = jj_consume_token(APOSTROPHE);
35 break;
36 case ACRONYM:
37 token = jj_consume_token(ACRONYM);
38 break;
39 case COMPANY:
40 token = jj_consume_token(COMPANY);
41 break;
42 case EMAIL:
43 token = jj_consume_token(EMAIL);
44 break;
45 case HOST:
46 token = jj_consume_token(HOST);
47 break;
48 case NUM:
49 token = jj_consume_token(NUM);
50 break;
51 case CJK:
52 token = jj_consume_token(CJK);
53 break;
54 case 0:
55 token = jj_consume_token(0);
56 break;
57 default:
58 jj_la1[0] = jj_gen;
59 jj_consume_token( -1);
60 throw new ParseException();
61 }
62 if (token.kind == EOF) {
63 {
64 if (true)return null;
65 }
66 }
67 else {
68 {
69 if (true)return
70 new org.apache.lucene.analysis.Token(token.image,
71 token.beginColumn, token.endColumn,
72 tokenImage[token.kind]);
73 }
74 }
75 }catch ( ParseException pe ){
76 throw new Error  ("ParseException occured ", pe);
77 }
78 throw new Error ("Missing return statement in function");
79 }
80
81 public StandardTokenizerTokenManager token_source;
82 public Token token, jj_nt;
83 private int jj_ntk;
84 private int jj_gen;
85 final private int[] jj_la1 = new int[1];
86 static private int[] jj_la1_0;
87 static {
88 jj_la1_0();
89 }
90 private static void jj_la1_0() {
91 jj_la1_0 = new int[] {0x10ff,};
92 }
93
94 public StandardTokenizer(CharStream stream) {
95 token_source = new StandardTokenizerTokenManager(stream);
96 token = new Token();
97 jj_ntk = -1;
98 jj_gen = 0;
99 for (int i = 0; i < 1; i++) jj_la1[i] = -1;
100 }
101
102 public void ReInit(CharStream stream) {
103 token_source.ReInit(stream);
104 token = new Token();
105 jj_ntk = -1;
106 jj_gen = 0;
107 for (int i = 0; i < 1; i++) jj_la1[i] = -1;
108 }
109
110 public StandardTokenizer(StandardTokenizerTokenManager tm) {
111 token_source = tm;
112 token = new Token();
113 jj_ntk = -1;
114 jj_gen = 0;
115 for (int i = 0; i < 1; i++) jj_la1[i] = -1;
116 }
117
118 public void ReInit(StandardTokenizerTokenManager tm) {
119 token_source = tm;
120 token = new Token();
121 jj_ntk = -1;
122 jj_gen = 0;
123 for (int i = 0; i < 1; i++) jj_la1[i] = -1;
124 }
125
126 final private Token jj_consume_token(int kind) throws ParseException {
127 Token oldToken;
128 if ((oldToken = token).next != null) token = token.next;
129 else token = token.next = token_source.getNextToken();
130 jj_ntk = -1;
131 if (token.kind == kind) {
132 jj_gen++;
133 return token;
134 }
135 token = oldToken;
136 jj_kind = kind;
137 throw generateParseException();
138 }
139
140 final public Token getNextToken() {
141 if (token.next != null) token = token.next;
142 else token = token.next = token_source.getNextToken();
143 jj_ntk = -1;
144 jj_gen++;
145 return token;
146 }
147
148 final public Token getToken(int index) {
149 Token t = token;
150 for (int i = 0; i < index; i++) {
151 if (t.next != null) t = t.next;
152 else t = t.next = token_source.getNextToken();
153 }
154 return t;
155 }
156
157 final private int jj_ntk() {
158 if ((jj_nt=token.next) == null)
159 return (jj_ntk = (token.next=token_source.getNextToken()).kind);
160 else
161 return (jj_ntk = jj_nt.kind);
162 }
163
164 private java.util.Vector jj_expentries = new java.util.Vector ();
165 private int[] jj_expentry;
166 private int jj_kind = -1;
167
168 public ParseException generateParseException() {
169 jj_expentries.removeAllElements();
170 boolean[] la1tokens = new boolean[15];
171 for (int i = 0; i < 15; i++) {
172 la1tokens[i] = false;
173 }
174 if (jj_kind >= 0) {
175 la1tokens[jj_kind] = true;
176 jj_kind = -1;
177 }
178 for (int i = 0; i < 1; i++) {
179 if (jj_la1[i] == jj_gen) {
180 for (int j = 0; j < 32; j++) {
181 if ((jj_la1_0[i] & (1<<j)) != 0) {
182 la1tokens[j] = true;
183 }
184 }
185 }
186 }
187 for (int i = 0; i < 15; i++) {
188 if (la1tokens[i]) {
189 jj_expentry = new int[1];
190 jj_expentry[0] = i;
191 jj_expentries.addElement(jj_expentry);
192 }
193 }
194 int[][] exptokseq = new int[jj_expentries.size()][];
195 for (int i = 0; i < jj_expentries.size(); i++) {
196 exptokseq[i] = (int[])jj_expentries.elementAt(i);
197 }
198 return new ParseException(token, exptokseq, tokenImage);
199 }
200
201 final public void enable_tracing() {
202 }
203
204 final public void disable_tracing() {
205 }
206
207 }
208 |
Popular Tags |
Java API By Example, From Geeks To Geeks.