KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > eclipse > help > internal > search > LowerCaseAndDigitsTokenizer


1 /*******************************************************************************
2  * Copyright (c) 2000, 2006 IBM Corporation and others.
3  * All rights reserved. This program and the accompanying materials
4  * are made available under the terms of the Eclipse Public License v1.0
5  * which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/epl-v10.html
7  *
8  * Contributors:
9  * IBM Corporation - initial API and implementation
10  *******************************************************************************/
11 package org.eclipse.help.internal.search;
12
13 import java.io.*;
14
15 import org.apache.lucene.analysis.*;
16
17 /**
18  * Tokenizer breaking words around letters or digits.
19  */
20 public class LowerCaseAndDigitsTokenizer extends CharTokenizer {
21
22     public LowerCaseAndDigitsTokenizer(Reader input) {
23         super(input);
24     }
25     protected char normalize(char c) {
26         return Character.toLowerCase(c);
27     }
28
29     protected boolean isTokenChar(char c) {
30         return Character.isLetterOrDigit(c);
31     }
32
33 }
34
Popular Tags