KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > opencms > search > extractors > A_CmsTextExtractor


1 /*
2  * File : $Source: /usr/local/cvs/opencms/src/org/opencms/search/extractors/A_CmsTextExtractor.java,v $
3  * Date : $Date: 2006/03/27 14:53:01 $
4  * Version: $Revision: 1.8 $
5  *
6  * This library is part of OpenCms -
7  * the Open Source Content Mananagement System
8  *
9  * Copyright (c) 2005 Alkacon Software GmbH (http://www.alkacon.com)
10  *
11  * This library is free software; you can redistribute it and/or
12  * modify it under the terms of the GNU Lesser General Public
13  * License as published by the Free Software Foundation; either
14  * version 2.1 of the License, or (at your option) any later version.
15  *
16  * This library is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19  * Lesser General Public License for more details.
20  *
21  * For further information about Alkacon Software GmbH, please see the
22  * company website: http://www.alkacon.com
23  *
24  * For further information about OpenCms, please see the
25  * project website: http://www.opencms.org
26  *
27  * You should have received a copy of the GNU Lesser General Public
28  * License along with this library; if not, write to the Free Software
29  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
30  */
31
32 package org.opencms.search.extractors;
33
34 import org.opencms.util.CmsFileUtil;
35 import org.opencms.util.CmsStringUtil;
36
37 import java.io.ByteArrayInputStream JavaDoc;
38 import java.io.IOException JavaDoc;
39 import java.io.InputStream JavaDoc;
40
41 /**
42  * Base utility class that allows extraction of the indexable "plain" text from a given document format.<p>
43  *
44  * @author Alexander Kandzior
45  *
46  * @version $Revision: 1.8 $
47  *
48  * @since 6.0.0
49  */
50 public abstract class A_CmsTextExtractor implements I_CmsTextExtractor {
51
52     /** A buffer in case the input stream must be read more then once. */
53     protected byte[] m_inputBuffer;
54
55     /**
56      * @see org.opencms.search.extractors.I_CmsTextExtractor#extractText(byte[])
57      */
58     public I_CmsExtractionResult extractText(byte[] content) throws Exception JavaDoc {
59
60         // encoding is null
61 return extractText(content, null);
62     }
63
64     /**
65      * @see org.opencms.search.extractors.I_CmsTextExtractor#extractText(byte[], java.lang.String)
66      */
67     public I_CmsExtractionResult extractText(byte[] content, String JavaDoc encoding) throws Exception JavaDoc {
68
69         // call stream based method of extraction
70 m_inputBuffer = content;
71         return extractText(new ByteArrayInputStream JavaDoc(content), encoding);
72     }
73
74     /**
75      * @see org.opencms.search.extractors.I_CmsTextExtractor#extractText(java.io.InputStream)
76      */
77     public I_CmsExtractionResult extractText(InputStream JavaDoc in) throws Exception JavaDoc {
78
79         // encoding is null
80 return extractText(in, null);
81     }
82
83     /**
84      * @see org.opencms.search.extractors.I_CmsTextExtractor#extractText(java.io.InputStream, java.lang.String)
85      */
86     public I_CmsExtractionResult extractText(InputStream JavaDoc in, String JavaDoc encoding) throws Exception JavaDoc {
87
88         // read the byte content
89 byte[] text = CmsFileUtil.readFully(in);
90         // call byte array based method of extraction
91 return extractText(text, encoding);
92     }
93
94     /**
95      * Creates a copy of the original input stream, which allows to read the input stream more then
96      * once, required for certain document types.<p>
97      *
98      * @param in the inpur stram to copy
99      * @return a copy of the original input stream
100      * @throws IOException in case of read errors from the original input stream
101      */
102     public InputStream JavaDoc getStreamCopy(InputStream JavaDoc in) throws IOException JavaDoc {
103
104         if (m_inputBuffer != null) {
105             return new ByteArrayInputStream JavaDoc(m_inputBuffer);
106         }
107
108         // read the input stream fully and copy it to a byte array
109 m_inputBuffer = CmsFileUtil.readFully(in);
110
111         // now return a reader from the byte array
112 return new ByteArrayInputStream JavaDoc(m_inputBuffer);
113     }
114
115     /**
116      * Removes "unwanted" control chars from the given content.<p>
117      *
118      * @param content the content to remove the unwanted control chars from
119      *
120      * @return the content with the unwanted control chars removed
121      */
122     protected String JavaDoc removeControlChars(String JavaDoc content) {
123
124         if (CmsStringUtil.isEmptyOrWhitespaceOnly(content)) {
125             // to avoid later null pointer exceptions an empty String is returned
126 return "";
127         }
128
129         char[] chars = content.toCharArray();
130         StringBuffer JavaDoc result = new StringBuffer JavaDoc(chars.length);
131         boolean wasUnwanted = false;
132         for (int i = 0; i < chars.length; i++) {
133             char ch = chars[i];
134
135             int type = Character.getType(ch);
136             switch (type) {
137
138                 // punctuation
139 case Character.CURRENCY_SYMBOL:
140                 case Character.CONNECTOR_PUNCTUATION:
141                 case Character.FINAL_QUOTE_PUNCTUATION:
142                 case Character.INITIAL_QUOTE_PUNCTUATION:
143                 case Character.DASH_PUNCTUATION:
144                 case Character.START_PUNCTUATION:
145                 case Character.END_PUNCTUATION:
146                 case Character.OTHER_PUNCTUATION:
147                 // letters
148 case Character.OTHER_LETTER:
149                 case Character.MODIFIER_LETTER:
150                 case Character.UPPERCASE_LETTER:
151                 case Character.TITLECASE_LETTER:
152                 case Character.LOWERCASE_LETTER:
153                 // digits
154 case Character.DECIMAL_DIGIT_NUMBER:
155                 // spaces
156 case Character.SPACE_SEPARATOR:
157                     result.append(ch);
158                     wasUnwanted = false;
159                     break;
160
161                 // line separators
162 case Character.LINE_SEPARATOR:
163                     result.append('\n');
164                     wasUnwanted = true;
165                     break;
166
167                 // symbols
168 case Character.MATH_SYMBOL:
169                 case Character.OTHER_SYMBOL:
170                 // other stuff:
171 case Character.CONTROL:
172                 case Character.COMBINING_SPACING_MARK:
173                 case Character.ENCLOSING_MARK:
174                 case Character.FORMAT:
175                 case Character.LETTER_NUMBER:
176                 case Character.MODIFIER_SYMBOL:
177                 case Character.NON_SPACING_MARK:
178                 case Character.PARAGRAPH_SEPARATOR:
179                 case Character.PRIVATE_USE:
180                 case Character.SURROGATE:
181                 case Character.UNASSIGNED:
182                 case Character.OTHER_NUMBER:
183                 default:
184                     if (!wasUnwanted) {
185                         result.append('\n');
186                         wasUnwanted = true;
187                     }
188             }
189         }
190
191         return result.toString();
192     }
193 }
Popular Tags