RuleBasedBreakIterator


1   /*
2    * @(#)RuleBasedBreakIterator.java  1.17 03/12/19
3    *
4    * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
5    * SUN PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
6    */
7   
8   /*
9    * @(#)RuleBasedBreakIterator.java  1.3 99/04/07
10   *
11   * (C) Copyright Taligent, Inc. 1996, 1997 - All Rights Reserved
12   * (C) Copyright IBM Corp. 1996 - 2002 - All Rights Reserved
13   *
14   * The original version of this source code and documentation
15   * is copyrighted and owned by Taligent, Inc., a wholly-owned
16   * subsidiary of IBM. These materials are provided under terms
17   * of a License Agreement between Taligent and Sun. This technology
18   * is protected by multiple US and International patents.
19   *
20   * This notice and attribution to Taligent may not be removed.
21   * Taligent is a registered trademark of Taligent, Inc.
22   */
23  
24  package java.text;
25  
26  import java.io.BufferedInputStream  ;
27  import java.io.IOException  ;
28  import java.security.AccessController  ;
29  import java.security.PrivilegedActionException  ;
30  import java.security.PrivilegedExceptionAction  ;
31  import java.util.Vector  ;
32  import java.util.Stack  ;
33  import java.util.Hashtable  ;
34  import java.util.Enumeration  ;
35  import java.util.MissingResourceException  ;
36  import java.text.CharacterIterator  ;
37  import java.text.StringCharacterIterator  ;
38  import sun.text.CompactByteArray;
39  import sun.text.SupplementaryCharacterData;
40  
41  /**
42   * <p>A subclass of BreakIterator whose behavior is specified using a list of rules.</p>
43   *
44   * <p>There are two kinds of rules, which are separated by semicolons: <i>substitutions</i>
45   * and <i>regular expressions.</i></p>
46   *
47   * <p>A substitution rule defines a name that can be used in place of an expression. It
48   * consists of a name, which is a string of characters contained in angle brackets, an equals
49   * sign, and an expression. (There can be no whitespace on either side of the equals sign.)
50   * To keep its syntactic meaning intact, the expression must be enclosed in parentheses or
51   * square brackets. A substitution is visible after its definition, and is filled in using
52   * simple textual substitution. Substitution definitions can contain other substitutions, as
53   * long as those substitutions have been defined first. Substitutions are generally used to
54   * make the regular expressions (which can get quite complex) shorted and easier to read.
55   * They typically define either character categories or commonly-used subexpressions.</p>
56   *
57   * <p>There is one special substitution.&nbsp; If the description defines a substitution
58   * called &quot;&lt;ignore&gt;&quot;, the expression must be a [] expression, and the
59   * expression defines a set of characters (the &quot;<em>ignore characters</em>&quot;) that
60   * will be transparent to the BreakIterator.&nbsp; A sequence of characters will break the
61   * same way it would if any ignore characters it contains are taken out.&nbsp; Break
62   * positions never occur befoer ignore characters.</p>
63   *
64   * <p>A regular expression uses a subset of the normal Unix regular-expression syntax, and
65   * defines a sequence of characters to be kept together. With one significant exception, the
66   * iterator uses a longest-possible-match algorithm when matching text to regular
67   * expressions. The iterator also treats descriptions containing multiple regular expressions
68   * as if they were ORed together (i.e., as if they were separated by |).</p>
69   *
70   * <p>The special characters recognized by the regular-expression parser are as follows:</p>
71   *
72   * <blockquote>
73   *   <table border="1" width="100%">
74   *     <tr>
75   *       <td width="6%">*</td>
76   *       <td width="94%">Specifies that the expression preceding the asterisk may occur any number
77   *       of times (including not at all).</td>
78   *     </tr>
79   *     <tr>
80   *       <td width="6%">{}</td>
81   *       <td width="94%">Encloses a sequence of characters that is optional.</td>
82   *     </tr>
83   *     <tr>
84   *       <td width="6%">()</td>
85   *       <td width="94%">Encloses a sequence of characters.&nbsp; If followed by *, the sequence
86   *       repeats.&nbsp; Otherwise, the parentheses are just a grouping device and a way to delimit
87   *       the ends of expressions containing |.</td>
88   *     </tr>
89   *     <tr>
90   *       <td width="6%">|</td>
91   *       <td width="94%">Separates two alternative sequences of characters.&nbsp; Either one
92   *       sequence or the other, but not both, matches this expression.&nbsp; The | character can
93   *       only occur inside ().</td>
94   *     </tr>
95   *     <tr>
96   *       <td width="6%">.</td>
97   *       <td width="94%">Matches any character.</td>
98   *     </tr>
99   *     <tr>
100  *       <td width="6%">*?</td>
101  *       <td width="94%">Specifies a non-greedy asterisk.&nbsp; *? works the same way as *, except
102  *       when there is overlap between the last group of characters in the expression preceding the
103  *       * and the first group of characters following the *.&nbsp; When there is this kind of
104  *       overlap, * will match the longest sequence of characters that match the expression before
105  *       the *, and *? will match the shortest sequence of characters matching the expression
106  *       before the *?.&nbsp; For example, if you have &quot;xxyxyyyxyxyxxyxyxyy&quot; in the text,
107  *       &quot;x[xy]*x&quot; will match through to the last x (i.e., &quot;<strong>xxyxyyyxyxyxxyxyx</strong>yy&quot;,
108  *       but &quot;x[xy]*?x&quot; will only match the first two xes (&quot;<strong>xx</strong>yxyyyxyxyxxyxyxyy&quot;).</td>
109  *     </tr>
110  *     <tr>
111  *       <td width="6%">[]</td>
112  *       <td width="94%">Specifies a group of alternative characters.&nbsp; A [] expression will
113  *       match any single character that is specified in the [] expression.&nbsp; For more on the
114  *       syntax of [] expressions, see below.</td>
115  *     </tr>
116  *     <tr>
117  *       <td width="6%">/</td>
118  *       <td width="94%">Specifies where the break position should go if text matches this
119  *       expression.&nbsp; (e.g., &quot;[a-z]&#42;/[:Zs:]*[1-0]&quot; will match if the iterator sees a run
120  *       of letters, followed by a run of whitespace, followed by a digit, but the break position
121  *       will actually go before the whitespace).&nbsp; Expressions that don't contain / put the
122  *       break position at the end of the matching text.</td>
123  *     </tr>
124  *     <tr>
125  *       <td width="6%">\</td>
126  *       <td width="94%">Escape character.&nbsp; The \ itself is ignored, but causes the next
127  *       character to be treated as literal character.&nbsp; This has no effect for many
128  *       characters, but for the characters listed above, this deprives them of their special
129  *       meaning.&nbsp; (There are no special escape sequences for Unicode characters, or tabs and
130  *       newlines; these are all handled by a higher-level protocol.&nbsp; In a Java string,
131  *       &quot;\n&quot; will be converted to a literal newline character by the time the
132  *       regular-expression parser sees it.&nbsp; Of course, this means that \ sequences that are
133  *       visible to the regexp parser must be written as \\ when inside a Java string.)&nbsp; All
134  *       characters in the ASCII range except for letters, digits, and control characters are
135  *       reserved characters to the parser and must be preceded by \ even if they currently don't
136  *       mean anything.</td>
137  *     </tr>
138  *     <tr>
139  *       <td width="6%">!</td>
140  *       <td width="94%">If ! appears at the beginning of a regular expression, it tells the regexp
141  *       parser that this expression specifies the backwards-iteration behavior of the iterator,
142  *       and not its normal iteration behavior.&nbsp; This is generally only used in situations
143  *       where the automatically-generated backwards-iteration brhavior doesn't produce
144  *       satisfactory results and must be supplemented with extra client-specified rules.</td>
145  *     </tr>
146  *     <tr>
147  *       <td width="6%"><em>(all others)</em></td>
148  *       <td width="94%">All other characters are treated as literal characters, which must match
149  *       the corresponding character(s) in the text exactly.</td>
150  *     </tr>
151  *   </table>
152  * </blockquote>
153  *
154  * <p>Within a [] expression, a number of other special characters can be used to specify
155  * groups of characters:</p>
156  *
157  * <blockquote>
158  *   <table border="1" width="100%">
159  *     <tr>
160  *       <td width="6%">-</td>
161  *       <td width="94%">Specifies a range of matching characters.&nbsp; For example
162  *       &quot;[a-p]&quot; matches all lowercase Latin letters from a to p (inclusive).&nbsp; The -
163  *       sign specifies ranges of continuous Unicode numeric values, not ranges of characters in a
164  *       language's alphabetical order: &quot;[a-z]&quot; doesn't include capital letters, nor does
165  *       it include accented letters such as a-umlaut.</td>
166  *     </tr>
167  *     <tr>
168  *       <td width="6%">::</td>
169  *       <td width="94%">A pair of colons containing a one- or two-letter code matches all
170  *       characters in the corresponding Unicode category.&nbsp; The two-letter codes are the same
171  *       as the two-letter codes in the Unicode database (for example, &quot;[:Sc::Sm:]&quot;
172  *       matches all currency symbols and all math symbols).&nbsp; Specifying a one-letter code is
173  *       the same as specifying all two-letter codes that begin with that letter (for example,
174  *       &quot;[:L:]&quot; matches all letters, and is equivalent to
175  *       &quot;[:Lu::Ll::Lo::Lm::Lt:]&quot;).&nbsp; Anything other than a valid two-letter Unicode
176  *       category code or a single letter that begins a Unicode category code is illegal within
177  *       colons.</td>
178  *     </tr>
179  *     <tr>
180  *       <td width="6%">[]</td>
181  *       <td width="94%">[] expressions can nest.&nbsp; This has no effect, except when used in
182  *       conjunction with the ^ token.</td>
183  *     </tr>
184  *     <tr>
185  *       <td width="6%">^</td>
186  *       <td width="94%">Excludes the character (or the characters in the [] expression) following
187  *       it from the group of characters.&nbsp; For example, &quot;[a-z^p]&quot; matches all Latin
188  *       lowercase letters except p.&nbsp; &quot;[:L:^[&#92;u4e00-&#92;u9fff]]&quot; matches all letters
189  *       except the Han ideographs.</td>
190  *     </tr>
191  *     <tr>
192  *       <td width="6%"><em>(all others)</em></td>
193  *       <td width="94%">All other characters are treated as literal characters.&nbsp; (For
194  *       example, &quot;[aeiou]&quot; specifies just the letters a, e, i, o, and u.)</td>
195  *     </tr>
196  *   </table>
197  * </blockquote>
198  *
199  * <p>For a more complete explanation, see <a
200  * HREF="http://www.ibm.com/java/education/boundaries/boundaries.html">http://www.ibm.com/java/education/boundaries/boundaries.html</a>.
201  * &nbsp; For examples, see the resource data (which is annotated).</p>
202  *
203  * @author Richard Gillam
204  * @version $RCSFile$ $Revision: 1.1 $ $Date: 1998/11/05 19:32:04 $
205  */
206 class RuleBasedBreakIterator extends BreakIterator   {
207 
208     /**
209      * A token used as a character-category value to identify ignore characters
210      */
211     protected static final byte IGNORE = -1;
212 
213     /**
214      * The state number of the starting state
215      */
216     private static final short START_STATE = 1;
217 
218     /**
219      * The state-transition value indicating "stop"
220      */
221     private static final short STOP_STATE = 0;
222 
223     /**
224      * Magic number for the BreakIterator data file format.
225      */
226     static final byte[] LABEL = {
227         (byte)'B', (byte)'I', (byte)'d', (byte)'a', (byte)'t', (byte)'a',
228         (byte)'\0'
229     };
230     static final int    LABEL_LENGTH = LABEL.length;
231 
232     /**
233      * Version number of the dictionary that was read in.
234      */
235     static final byte supportedVersion = 1;
236 
237     /**
238      * Header size in byte count
239      */
240     private static final int HEADER_LENGTH = 36;
241 
242     /**
243      * An array length of indices for BMP characters
244      */
245     private static final int BMP_INDICES_LENGTH = 512;
246 
247     /**
248      * Tables that indexes from character values to character category numbers
249      */
250     private CompactByteArray charCategoryTable = null;
251     private SupplementaryCharacterData supplementaryCharCategoryTable = null;
252 
253     /**
254      * The table of state transitions used for forward iteration
255      */
256     private short[] stateTable = null;
257 
258     /**
259      * The table of state transitions used to sync up the iterator with the
260      * text in backwards and random-access iteration
261      */
262     private short[] backwardsStateTable = null;
263 
264     /**
265      * A list of flags indicating which states in the state table are accepting
266      * ("end") states
267      */
268     private boolean[] endStates = null;
269 
270     /**
271      * A list of flags indicating which states in the state table are
272      * lookahead states (states which turn lookahead on and off)
273      */
274     private boolean[] lookaheadStates = null;
275 
276     /**
277      * A table for additional data. May be used by a subclass of
278      * RuleBasedBreakIterator.
279      */
280     private byte[] additionalData = null;
281 
282     /**
283      * The number of character categories (and, thus, the number of columns in
284      * the state tables)
285      */
286     private int numCategories;
287 
288     /**
289      * The character iterator through which this BreakIterator accesses the text
290      */
291     private CharacterIterator   text = null;
292 
293     /**
294      * A CRC32 value of all data in datafile
295      */
296     private long checksum;
297 
298     //=======================================================================
299     // constructors
300     //=======================================================================
301 
302     /**
303      * Constructs a RuleBasedBreakIterator according to the datafile
304      * provided.
305      */
306     public RuleBasedBreakIterator(String   datafile)
307         throws IOException  , MissingResourceException   {
308         readTables(datafile);
309     }
310 
311     /**
312      * Read datafile. The datafile's format is as follows:
313      * <pre>
314      *   BreakIteratorData {
315      *       u1           magic[7];
316      *       u1           version;
317      *       u4           totalDataSize;
318      *       header_info  header;
319      *       body         value;
320      *   }
321      * </pre>
322      * <code>totalDataSize</code> is the summation of the size of
323      * <code>header_info</code> and <code>body</code> in byte count.
324      * <p>
325      * In <code>header</code>, each field except for checksum implies the
326      * length of each field. Since <code>BMPdataLength</code> is a fixed-length
327      *  data(512 entries), its length isn't included in <code>header</code>.
328      * <code>checksum</code> is a CRC32 value of all in <code>body</code>.
329      * <pre>
330      *   header_info { 
331      *       u4           stateTableLength;
332      *       u4           backwardsStateTableLength;
333      *       u4           endStatesLength;
334      *       u4           lookaheadStatesLength;
335      *       u4           BMPdataLength;
336      *       u4           nonBMPdataLength;
337      *       u4           additionalDataLength;
338      *       u8           checksum;
339      *   }
340      * </pre>
341      * <p>
342      * 
343      * Finally, <code>BMPindices</code> and <code>BMPdata</code> are set to
344      * <code>charCategoryTable</code>. <code>nonBMPdata</code> is set to
345      * <code>supplementaryCharCategoryTable</code>.
346      * <pre>
347      *   body {
348      *       u2           stateTable[stateTableLength];
349      *       u2           backwardsStateTable[backwardsStateTableLength];
350      *       u1           endStates[endStatesLength];
351      *       u1           lookaheadStates[lookaheadStatesLength];
352      *       u2           BMPindices[512];
353      *       u1           BMPdata[BMPdataLength];
354      *       u4           nonBMPdata[numNonBMPdataLength];
355      *       u1           additionalData[additionalDataLength];
356      *   }
357      * </pre>
358      */
359     protected void readTables(String   datafile)
360         throws IOException  , MissingResourceException   {
361 
362         byte[] buffer = readFile(datafile);
363 
364         /* Read header_info. */
365         int stateTableLength = BreakIterator.getInt(buffer, 0);
366         int backwardsStateTableLength = BreakIterator.getInt(buffer, 4);
367         int endStatesLength = BreakIterator.getInt(buffer, 8);
368         int lookaheadStatesLength = BreakIterator.getInt(buffer, 12);
369         int BMPdataLength = BreakIterator.getInt(buffer, 16);
370         int nonBMPdataLength = BreakIterator.getInt(buffer, 20);
371         int additionalDataLength = BreakIterator.getInt(buffer, 24);
372         checksum = BreakIterator.getLong(buffer, 28);
373 
374         /* Read stateTable[numCategories * numRows] */
375         stateTable = new short[stateTableLength];
376         int offset = HEADER_LENGTH;
377         for (int i = 0; i < stateTableLength; i++, offset+=2) {
378            stateTable[i] = BreakIterator.getShort(buffer, offset);
379         }
380 
381         /* Read backwardsStateTable[numCategories * numRows] */
382         backwardsStateTable = new short[backwardsStateTableLength];
383         for (int i = 0; i < backwardsStateTableLength; i++, offset+=2) {
384            backwardsStateTable[i] = BreakIterator.getShort(buffer, offset);
385         }
386 
387         /* Read endStates[numRows] */
388         endStates = new boolean[endStatesLength];
389         for (int i = 0; i < endStatesLength; i++, offset++) {
390            endStates[i] = buffer[offset] == 1; 
391         }
392 
393         /* Read lookaheadStates[numRows] */
394         lookaheadStates = new boolean[lookaheadStatesLength];
395         for (int i = 0; i < lookaheadStatesLength; i++, offset++) {
396            lookaheadStates[i] = buffer[offset] == 1; 
397         }
398 
399         /* Read a category table and indices for BMP characters. */
400         short[] temp1 = new short[BMP_INDICES_LENGTH];  // BMPindices
401         for (int i = 0; i < BMP_INDICES_LENGTH; i++, offset+=2) {
402             temp1[i] = BreakIterator.getShort(buffer, offset);
403         }
404         byte[] temp2 = new byte[BMPdataLength];  // BMPdata
405         System.arraycopy(buffer, offset, temp2, 0, BMPdataLength);
406         offset += BMPdataLength;
407         charCategoryTable = new CompactByteArray(temp1, temp2);
408 
409         /* Read a category table for non-BMP characters. */
410         int[] temp3 = new int[nonBMPdataLength];
411         for (int i = 0; i < nonBMPdataLength; i++, offset+=4) {
412             temp3[i] = BreakIterator.getInt(buffer, offset);
413         }
414         supplementaryCharCategoryTable = new SupplementaryCharacterData(temp3);
415 
416         /* Read additional data */ 
417         if (additionalDataLength > 0) {
418             additionalData = new byte[additionalDataLength];
419             System.arraycopy(buffer, offset, additionalData, 0, additionalDataLength);
420         }
421 
422         /* Set numCategories */
423         numCategories = stateTable.length / endStates.length;
424     }
425 
426     protected byte[] readFile(final String   datafile)
427         throws IOException  , MissingResourceException   {
428 
429         BufferedInputStream   is;
430         try {
431             is = (BufferedInputStream  )AccessController.doPrivileged(
432                 new PrivilegedExceptionAction  () {
433                     public Object   run() throws Exception   {
434                         return new BufferedInputStream  (getClass().getResourceAsStream("/sun/text/resources/" + datafile));
435                     }
436                 }
437             );
438         }                
439         catch (PrivilegedActionException   e) {
440             throw new InternalError  (e.toString());
441         }
442 
443         int offset = 0;
444 
445         /* First, read magic, version, and header_info. */
446         int len = LABEL_LENGTH + 5;
447         byte[] buf = new byte[len];
448         if (is.read(buf) != len) {
449             throw new MissingResourceException  ("Wrong header length",
450                                                datafile, "");
451         }
452 
453         /* Validate the magic number. */
454         for (int i = 0; i < LABEL_LENGTH; i++, offset++) {
455             if (buf[offset] != LABEL[offset]) {
456                 throw new MissingResourceException  ("Wrong magic number",
457                                                    datafile, "");
458             }
459         }
460 
461         /* Validate the version number. */
462         if (buf[offset] != supportedVersion) {
463             throw new MissingResourceException  ("Unsupported version(" + buf[offset] + ")",
464                                                datafile, "");
465         }
466 
467         /* Read data: totalDataSize + 8(for checksum) */
468         len = BreakIterator.getInt(buf, ++offset);
469         buf = new byte[len];
470         if (is.read(buf) != len) {
471             throw new MissingResourceException  ("Wrong data length",
472                                                datafile, "");
473         }
474 
475         is.close();
476 
477         return buf;
478     }
479 
480     byte[] getAdditionalData() {
481         return additionalData;
482     }
483 
484     void setAdditionalData(byte[] b) {
485         additionalData = b;
486     }
487 
488     //=======================================================================
489     // boilerplate
490     //=======================================================================
491     /**
492      * Clones this iterator.
493      * @return A newly-constructed RuleBasedBreakIterator with the same
494      * behavior as this one.
495      */
496     public Object   clone() {
497         RuleBasedBreakIterator   result = (RuleBasedBreakIterator  ) super.clone();
498         if (text != null) {
499             result.text = (CharacterIterator  ) text.clone();
500         }
501         return result;
502     }
503 
504     /**
505      * Returns true if both BreakIterators are of the same class, have the same
506      * rules, and iterate over the same text.
507      */
508     public boolean equals(Object   that) {
509         try {
510             if (that == null) {
511                 return false;
512             }
513 
514             RuleBasedBreakIterator   other = (RuleBasedBreakIterator  ) that;
515             if (checksum != other.checksum) {
516                 return false;
517             }
518             if (text == null) {
519                 return other.text == null;
520             } else {
521                 return text.equals(other.text);
522             }
523         }
524         catch(ClassCastException   e) {
525             return false;
526         }
527     }
528 
529     /**
530      * Returns text
531      */
532     public String   toString() {
533         StringBuffer   sb = new StringBuffer  ();
534         sb.append('[');
535         sb.append("checksum=0x" + Long.toHexString(checksum));
536         sb.append(']');
537         return sb.toString();
538     }
539 
540     /**
541      * Compute a hashcode for this BreakIterator
542      * @return A hash code
543      */
544     public int hashCode() {
545         return (int)checksum;
546     }
547 
548     //=======================================================================
549     // BreakIterator overrides
550     //=======================================================================
551 
552     /**
553      * Sets the current iteration position to the beginning of the text.
554      * (i.e., the CharacterIterator's starting offset).
555      * @return The offset of the beginning of the text.
556      */
557     public int first() {
558         CharacterIterator   t = getText();
559 
560         t.first();
561         return t.getIndex();
562     }
563 
564     /**
565      * Sets the current iteration position to the end of the text.
566      * (i.e., the CharacterIterator's ending offset).
567      * @return The text's past-the-end offset.
568      */
569     public int last() {
570         CharacterIterator   t = getText();
571 
572         // I'm not sure why, but t.last() returns the offset of the last character,
573         // rather than the past-the-end offset
574         t.setIndex(t.getEndIndex());
575         return t.getIndex();
576     }
577 
578     /**
579      * Advances the iterator either forward or backward the specified number of steps.
580      * Negative values move backward, and positive values move forward.  This is
581      * equivalent to repeatedly calling next() or previous().
582      * @param n The number of steps to move.  The sign indicates the direction
583      * (negative is backwards, and positive is forwards).
584      * @return The character offset of the boundary position n boundaries away from
585      * the current one.
586      */
587     public int next(int n) {
588         int result = current();
589         while (n > 0) {
590             result = handleNext();
591             --n;
592         }
593         while (n < 0) {
594             result = previous();
595             ++n;
596         }
597         return result;
598     }
599 
600     /**
601      * Advances the iterator to the next boundary position.
602      * @return The position of the first boundary after this one.
603      */
604     public int next() {
605         return handleNext();
606     }
607 
608     /**
609      * Advances the iterator backwards, to the last boundary preceding this one.
610      * @return The position of the last boundary position preceding this one.
611      */
612     public int previous() {
613         // if we're already sitting at the beginning of the text, return DONE
614         CharacterIterator   text = getText();
615         if (current() == text.getBeginIndex()) {
616             return BreakIterator.DONE;
617         }
618 
619         // set things up.  handlePrevious() will back us up to some valid
620         // break position before the current position (we back our internal
621         // iterator up one step to prevent handlePrevious() from returning
622         // the current position), but not necessarily the last one before
623         // where we started
624         int start = current();
625         getPrevious();
626         int lastResult = handlePrevious();
627         int result = lastResult;
628 
629         // iterate forward from the known break position until we pass our
630         // starting point.  The last break position before the starting
631         // point is our return value
632         while (result != BreakIterator.DONE && result < start) {
633             lastResult = result;
634             result = handleNext();
635         }
636 
637         // set the current iteration position to be the last break position
638         // before where we started, and then return that value
639         text.setIndex(lastResult);
640         return lastResult;
641     }
642 
643     /**
644      * Returns previous character
645      */
646     private int getPrevious() {
647         char c2 = text.previous();
648         if (Character.isLowSurrogate(c2) &&
649             text.getIndex() > text.getBeginIndex()) {
650             char c1 = text.previous();
651             if (Character.isHighSurrogate(c1)) {
652                 return Character.toCodePoint(c1, c2);
653             } else {
654                 text.next();
655             }
656         }
657         return (int)c2;
658     }
659 
660     /**
661      * Returns current character
662      */
663     int getCurrent() {
664         char c1 = text.current();
665         if (Character.isHighSurrogate(c1) &&
666             text.getIndex() < text.getEndIndex()) {
667             char c2 = text.next();
668             text.previous();
669             if (Character.isLowSurrogate(c2)) {
670                 return Character.toCodePoint(c1, c2);
671             }
672         }
673         return (int)c1;
674     }
675 
676     /**
677      * Returns the count of next character.
678      */
679     private int getCurrentCodePointCount() {
680         char c1 = text.current();
681         if (Character.isHighSurrogate(c1) &&
682             text.getIndex() < text.getEndIndex()) {
683             char c2 = text.next();
684             text.previous();
685             if (Character.isLowSurrogate(c2)) {
686                 return 2;
687             }
688         }
689         return 1;
690     }
691 
692     /**
693      * Returns next character
694      */
695     int getNext() {
696         int index = text.getIndex();
697         int endIndex = text.getEndIndex();
698         if (index == endIndex ||
699             (index = index + getCurrentCodePointCount()) >= endIndex) {
700             return CharacterIterator.DONE;
701         }
702         text.setIndex(index);
703         return getCurrent();
704     }
705 
706     /**
707      * Returns the position of next character.
708      */
709     private int getNextIndex() {
710         int index = text.getIndex() + getCurrentCodePointCount();
711         int endIndex = text.getEndIndex();
712         if (index > endIndex) {
713             return endIndex;
714         } else {
715             return index;
716         }
717     }
718 
719     /**
720      * Throw IllegalArgumentException unless begin <= offset < end.
721      */
722     protected static final void checkOffset(int offset, CharacterIterator   text) {
723         if (offset < text.getBeginIndex() || offset >= text.getEndIndex()) {
724             throw new IllegalArgumentException  ("offset out of bounds");
725         }
726     }
727 
728     /**
729      * Sets the iterator to refer to the first boundary position following
730      * the specified position.
731      * @offset The position from which to begin searching for a break position.
732      * @return The position of the first break after the current position.
733      */
734     public int following(int offset) {
735 
736         CharacterIterator   text = getText();
737         checkOffset(offset, text);
738 
739         // Set our internal iteration position (temporarily)
740         // to the position passed in.  If this is the _beginning_ position,
741         // then we can just use next() to get our return value
742         text.setIndex(offset);
743         if (offset == text.getBeginIndex()) {
744             return handleNext();
745         }
746 
747         // otherwise, we have to sync up first.  Use handlePrevious() to back
748         // us up to a known break position before the specified position (if
749         // we can determine that the specified position is a break position,
750         // we don't back up at all).  This may or may not be the last break
751         // position at or before our starting position.  Advance forward
752         // from here until we've passed the starting position.  The position
753         // we stop on will be the first break position after the specified one.
754         int result = handlePrevious();
755         while (result != BreakIterator.DONE && result <= offset) {
756             result = handleNext();
757         }
758         return result;
759     }
760 
761     /**
762      * Sets the iterator to refer to the last boundary position before the
763      * specified position.
764      * @offset The position to begin searching for a break from.
765      * @return The position of the last boundary before the starting position.
766      */
767     public int preceding(int offset) {
768         // if we start by updating the current iteration position to the
769         // position specified by the caller, we can just use previous()
770         // to carry out this operation
771         CharacterIterator   text = getText();
772         checkOffset(offset, text);
773         text.setIndex(offset);
774         return previous();
775     }
776 
777     /**
778      * Returns true if the specfied position is a boundary position.  As a side
779      * effect, leaves the iterator pointing to the first boundary position at
780      * or after "offset".
781      * @param offset the offset to check.
782      * @return True if "offset" is a boundary position.
783      */
784     public boolean isBoundary(int offset) {
785         CharacterIterator   text = getText();
786         checkOffset(offset, text);
787         if (offset == text.getBeginIndex()) {
788             return true;
789         }
790 
791         // to check whether this is a boundary, we can use following() on the
792         // position before the specified one and return true if the position we
793         // get back is the one the user specified
794         else {
795             return following(offset - 1) == offset;
796         }
797     }
798 
799     /**
800      * Returns the current iteration position.
801      * @return The current iteration position.
802      */
803     public int current() {
804         return getText().getIndex();
805     }
806 
807     /**
808      * Return a CharacterIterator over the text being analyzed.  This version
809      * of this method returns the actual CharacterIterator we're using internally.
810      * Changing the state of this iterator can have undefined consequences.  If
811      * you need to change it, clone it first.
812      * @return An iterator over the text being analyzed.
813      */
814     public CharacterIterator   getText() {
815         // The iterator is initialized pointing to no text at all, so if this
816         // function is called while we're in that state, we have to fudge an
817         // iterator to return.
818         if (text == null) {
819             text = new StringCharacterIterator  ("");
820         }
821         return text;
822     }
823 
824     /**
825      * Set the iterator to analyze a new piece of text.  This function resets
826      * the current iteration position to the beginning of the text.
827      * @param newText An iterator over the text to analyze.
828      */
829     public void setText(CharacterIterator   newText) {
830         // Test iterator to see if we need to wrap it in a SafeCharIterator.
831         // The correct behavior for CharacterIterators is to allow the
832         // position to be set to the endpoint of the iterator.  Many
833         // CharacterIterators do not uphold this, so this is a workaround
834         // to permit them to use this class.
835         int end = newText.getEndIndex();
836         boolean goodIterator;
837         try {
838             newText.setIndex(end);  // some buggy iterators throw an exception here
839             goodIterator = newText.getIndex() == end;
840         }
841         catch(IllegalArgumentException   e) {
842             goodIterator = false;
843         }
844 
845         if (goodIterator) {
846             text = newText;
847         }
848         else {
849             text = new SafeCharIterator(newText);
850         }
851         text.first();
852     }
853 
854 
855     //=======================================================================
856     // implementation
857     //=======================================================================
858 
859     /**
860      * This method is the actual implementation of the next() method.  All iteration
861      * vectors through here.  This method initializes the state machine to state 1
862      * and advances through the text character by character until we reach the end
863      * of the text or the state machine transitions to state 0.  We update our return
864      * value every time the state machine passes through a possible end state.
865      */
866     protected int handleNext() {
867         // if we're already at the end of the text, return DONE.
868         CharacterIterator   text = getText();
869         if (text.getIndex() == text.getEndIndex()) {
870             return BreakIterator.DONE;
871         }
872 
873         // no matter what, we always advance at least one character forward
874         int result = getNextIndex();
875         int lookaheadResult = 0;
876 
877         // begin in state 1
878         int state = START_STATE;
879         int category;
880         int c = getCurrent();
881 
882         // loop until we reach the end of the text or transition to state 0
883         while (c != CharacterIterator.DONE && state != STOP_STATE) {
884 
885             // look up the current character's character category (which tells us
886             // which column in the state table to look at)
887             category = lookupCategory(c);
888 
889             // if the character isn't an ignore character, look up a state
890             // transition in the state table
891             if (category != IGNORE) {
892                 state = lookupState(state, category);
893             }
894 
895             // if the state we've just transitioned to is a lookahead state,
896             // (but not also an end state), save its position.  If it's
897             // both a lookahead state and an end state, update the break position
898             // to the last saved lookup-state position
899             if (lookaheadStates[state]) {
900                 if (endStates[state]) {
901                     result = lookaheadResult;
902                 }
903                 else {
904                     lookaheadResult = getNextIndex();
905                 }
906             }
907 
908             // otherwise, if the state we've just transitioned to is an accepting
909             // state, update the break position to be the current iteration position
910             else {
911                 if (endStates[state]) {
912                     result = getNextIndex();
913                 }
914             }
915 
916             c = getNext();
917         }
918 
919         // if we've run off the end of the text, and the very last character took us into
920         // a lookahead state, advance the break position to the lookahead position
921         // (the theory here is that if there are no characters at all after the lookahead
922         // position, that always matches the lookahead criteria)
923         if (c == CharacterIterator.DONE && lookaheadResult == text.getEndIndex()) {
924             result = lookaheadResult;
925         }
926 
927         text.setIndex(result);
928         return result;
929     }
930 
931     /**
932      * This method backs the iterator back up to a "safe position" in the text.
933      * This is a position that we know, without any context, must be a break position.
934      * The various calling methods then iterate forward from this safe position to
935      * the appropriate position to return.  (For more information, see the description
936      * of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.)
937      */
938     protected int handlePrevious() {
939         CharacterIterator   text = getText();
940         int state = START_STATE;
941         int category = 0;
942         int lastCategory = 0;
943         int c = getCurrent();
944 
945         // loop until we reach the beginning of the text or transition to state 0
946         while (c != CharacterIterator.DONE && state != STOP_STATE) {
947 
948             // save the last character's category and look up the current
949             // character's category
950             lastCategory = category;
951             category = lookupCategory(c);
952 
953             // if the current character isn't an ignore character, look up a
954             // state transition in the backwards state table
955             if (category != IGNORE) {
956                 state = lookupBackwardState(state, category);
957             }
958 
959             // then advance one character backwards
960             c = getPrevious();
961         }
962 
963         // if we didn't march off the beginning of the text, we're either one or two
964         // positions away from the real break position.  (One because of the call to
965         // previous() at the end of the loop above, and another because the character
966         // that takes us into the stop state will always be the character BEFORE
967         // the break position.)
968         if (c != CharacterIterator.DONE) {
969             if (lastCategory != IGNORE) {
970                 getNext();
971                 getNext();
972             }
973             else {
974                 getNext();
975             }
976         }
977         return text.getIndex();
978     }
979 
980     /**
981      * Looks up a character's category (i.e., its category for breaking purposes,
982      * not its Unicode category)
983      */
984     protected int lookupCategory(int c) {
985         if (c < Character.MIN_SUPPLEMENTARY_CODE_POINT) {
986             return charCategoryTable.elementAt((char)c);
987         } else {
988             return supplementaryCharCategoryTable.getValue(c);
989         }
990     }
991 
992     /**
993      * Given a current state and a character category, looks up the
994      * next state to transition to in the state table.
995      */
996     protected int lookupState(int state, int category) {
997         return stateTable[state * numCategories + category];
998     }
999 
1000    /**
1001     * Given a current state and a character category, looks up the
1002     * next state to transition to in the backwards state table.
1003     */
1004    protected int lookupBackwardState(int state, int category) {
1005        return backwardsStateTable[state * numCategories + category];
1006    }
1007
1008    /*
1009     * This class exists to work around a bug in incorrect implementations
1010     * of CharacterIterator, which incorrectly handle setIndex(endIndex).
1011     * This iterator relies only on base.setIndex(n) where n is less than
1012     * endIndex.
1013     *
1014     * One caveat:  if the base iterator's begin and end indices change
1015     * the change will not be reflected by this wrapper.  Does that matter?
1016     */
1017    private static final class SafeCharIterator implements CharacterIterator  ,
1018                                                           Cloneable   {
1019
1020        private CharacterIterator   base;
1021        private int rangeStart;
1022        private int rangeLimit;
1023        private int currentIndex;
1024
1025        SafeCharIterator(CharacterIterator   base) {
1026            this.base = base;
1027            this.rangeStart = base.getBeginIndex();
1028            this.rangeLimit = base.getEndIndex();
1029            this.currentIndex = base.getIndex();
1030        }
1031
1032        public char first() {
1033            return setIndex(rangeStart);
1034        }
1035
1036        public char last() {
1037            return setIndex(rangeLimit - 1);
1038        }
1039
1040        public char current() {
1041            if (currentIndex < rangeStart || currentIndex >= rangeLimit) {
1042                return DONE;
1043            }
1044            else {
1045                return base.setIndex(currentIndex);
1046            }
1047        }
1048
1049        public char next() {
1050
1051            currentIndex++;
1052            if (currentIndex >= rangeLimit) {
1053                currentIndex = rangeLimit;
1054                return DONE;
1055            }
1056            else {
1057                return base.setIndex(currentIndex);
1058            }
1059        }
1060
1061        public char previous() {
1062
1063            currentIndex--;
1064            if (currentIndex < rangeStart) {
1065                currentIndex = rangeStart;
1066                return DONE;
1067            }
1068            else {
1069                return base.setIndex(currentIndex);
1070            }
1071        }
1072
1073        public char setIndex(int i) {
1074
1075            if (i < rangeStart || i > rangeLimit) {
1076                throw new IllegalArgumentException  ("Invalid position");
1077            }
1078            currentIndex = i;
1079            return current();
1080        }
1081
1082        public int getBeginIndex() {
1083            return rangeStart;
1084        }
1085
1086        public int getEndIndex() {
1087            return rangeLimit;
1088        }
1089
1090        public int getIndex() {
1091            return currentIndex;
1092        }
1093
1094        public Object   clone() {
1095
1096            SafeCharIterator copy = null;
1097            try {
1098                copy = (SafeCharIterator) super.clone();
1099            }
1100            catch(CloneNotSupportedException   e) {
1101                throw new Error  ("Clone not supported: " + e);
1102            }
1103
1104            CharacterIterator   copyOfBase = (CharacterIterator  ) base.clone();
1105            copy.base = copyOfBase;
1106            return copy;
1107        }
1108    }
1109}
1110
1111
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags