StringPrep


1   /*
2    *******************************************************************************
3    * Copyright (C) 2003-2005, International Business Machines Corporation and         *
4    * others. All Rights Reserved.                                                *
5    *******************************************************************************
6    */
7   package com.ibm.icu.text;
8   
9   import java.io.BufferedInputStream  ;
10  import java.io.ByteArrayInputStream  ;
11  import java.io.IOException  ;
12  import java.io.InputStream  ;
13  
14  import com.ibm.icu.impl.CharTrie;
15  import com.ibm.icu.impl.StringPrepDataReader;
16  import com.ibm.icu.impl.Trie;
17  import com.ibm.icu.impl.NormalizerImpl;
18  import com.ibm.icu.impl.UBiDiProps;
19  
20  import com.ibm.icu.util.VersionInfo;
21  
22  import com.ibm.icu.lang.UCharacter;
23  import com.ibm.icu.lang.UCharacterDirection;
24  
25  /**
26   * StringPrep API implements the StingPrep framework as described by 
27   * <a HREF="http://www.ietf.org/rfc/rfc3454.txt">RFC 3454</a>.
28   * StringPrep prepares Unicode strings for use in network protocols.
29   * Profiles of StingPrep are set of rules and data according to which the
30   * Unicode Strings are prepared. Each profiles contains tables which describe
31   * how a code point should be treated. The tables are broadly classied into
32   * <ul>
33   *     <li> Unassigned Table: Contains code points that are unassigned 
34   *          in the Unicode Version supported by StringPrep. Currently 
35   *          RFC 3454 supports Unicode 3.2. </li>
36   *     <li> Prohibited Table: Contains code points that are prohibted from
37   *          the output of the StringPrep processing function. </li>
38   *     <li> Mapping Table: Contains code ponts that are deleted from the output or case mapped. </li>
39   * </ul>
40   * 
41   * The procedure for preparing Unicode strings:
42   * <ol>
43   *      <li> Map: For each character in the input, check if it has a mapping
44   *           and, if so, replace it with its mapping. </li>
45   *      <li> Normalize: Possibly normalize the result of step 1 using Unicode
46   *           normalization. </li>
47   *      <li> Prohibit: Check for any characters that are not allowed in the
48   *           output.  If any are found, return an error.</li>
49   *      <li> Check bidi: Possibly check for right-to-left characters, and if
50   *           any are found, make sure that the whole string satisfies the
51   *           requirements for bidirectional strings.  If the string does not
52   *           satisfy the requirements for bidirectional strings, return an
53   *           error.  </li>
54   * </ol>
55   * @author Ram Viswanadha
56   * @stable ICU 2.8
57   */
58  public final class StringPrep {
59      /** 
60       * Option to prohibit processing of unassigned code points in the input
61       * 
62       * @see   #prepare
63       * @stable ICU 2.8
64       */
65      public static final int DEFAULT = 0x0000;
66  
67      /** 
68       * Option to allow processing of unassigned code points in the input
69       * 
70       * @see   #prepare
71       * @stable ICU 2.8
72       */
73      public static final int ALLOW_UNASSIGNED = 0x0001;
74      
75      private static final int UNASSIGNED        = 0x0000; 
76      private static final int MAP               = 0x0001; 
77      private static final int PROHIBITED        = 0x0002; 
78      private static final int DELETE            = 0x0003;
79      private static final int TYPE_LIMIT        = 0x0004;
80      
81      private static final int NORMALIZATION_ON  = 0x0001;
82      private static final int CHECK_BIDI_ON     = 0x0002;
83      
84      private static final int TYPE_THRESHOLD       = 0xFFF0;
85      private static final int MAX_INDEX_VALUE      = 0x3FBF;   /*16139*/ 
86      private static final int MAX_INDEX_TOP_LENGTH = 0x0003;
87      
88      /* indexes[] value names */
89      private static final int INDEX_TRIE_SIZE                  =  0; /* number of bytes in normalization trie */
90      private static final int INDEX_MAPPING_DATA_SIZE          =  1; /* The array that contains the mapping   */
91      private static final int NORM_CORRECTNS_LAST_UNI_VERSION  =  2; /* The index of Unicode version of last entry in NormalizationCorrections.txt */ 
92      private static final int ONE_UCHAR_MAPPING_INDEX_START    =  3; /* The starting index of 1 UChar mapping index in the mapping data array */
93      private static final int TWO_UCHARS_MAPPING_INDEX_START   =  4; /* The starting index of 2 UChars mapping index in the mapping data array */
94      private static final int THREE_UCHARS_MAPPING_INDEX_START =  5;
95      private static final int FOUR_UCHARS_MAPPING_INDEX_START  =  6;
96      private static final int OPTIONS                          =  7; /* Bit set of options to turn on in the profile */
97      private static final int INDEX_TOP                        = 16;                          /* changing this requires a new formatVersion */
98     
99     
100     /**
101      * Default buffer size of datafile
102      */
103     private static final int DATA_BUFFER_SIZE = 25000;
104     
105     // CharTrie implmentation for reading the trie data
106     private CharTrie sprepTrie;
107     // Indexes read from the data file
108     private int[] indexes;
109     // mapping data read from the data file
110     private char[] mappingData;
111     // format version of the data file
112     private byte[] formatVersion;
113     // the version of Unicode supported by the data file
114     private VersionInfo sprepUniVer;
115     // the Unicode version of last entry in the
116     // NormalizationCorrections.txt file if normalization
117     // is turned on 
118     private VersionInfo normCorrVer;
119     // Option to turn on Normalization
120     private boolean doNFKC;
121     // Option to turn on checking for BiDi rules
122     private boolean checkBiDi;
123     // bidi properties
124     private UBiDiProps bdp;
125     
126     private char getCodePointValue(int ch){
127         return sprepTrie.getCodePointValue(ch);
128     }
129   
130     private static VersionInfo getVersionInfo(int comp){
131         int micro = comp & 0xFF;
132         int milli =(comp >> 8)  & 0xFF;
133         int minor =(comp >> 16) & 0xFF;
134         int major =(comp >> 24) & 0xFF;
135         return VersionInfo.getInstance(major,minor,milli,micro);
136     }
137     private static VersionInfo getVersionInfo(byte[] version){
138         if(version.length != 4){
139             return null;
140         }
141         return VersionInfo.getInstance((int)version[0],(int) version[1],(int) version[2],(int) version[3]);
142     }
143     /**
144      * Creates an StringPrep object after reading the input stream.
145      * The object does not hold a reference to the input steam, so the stream can be
146      * closed after the method returns.
147      * 
148      * @param inputStream The stream for reading the StringPrep profile binarySun 
149      * @throws IOException
150      * @stable ICU 2.8
151      */
152     public StringPrep(InputStream   inputStream) throws IOException  {
153 
154         BufferedInputStream   b = new BufferedInputStream  (inputStream,DATA_BUFFER_SIZE);
155   
156         StringPrepDataReader reader = new StringPrepDataReader(b);
157         
158         // read the indexes            
159         indexes = reader.readIndexes(INDEX_TOP);
160    
161         byte[] sprepBytes = new byte[indexes[INDEX_TRIE_SIZE]];
162    
163 
164         //indexes[INDEX_MAPPING_DATA_SIZE] store the size of mappingData in bytes           
165         mappingData = new char[indexes[INDEX_MAPPING_DATA_SIZE]/2]; 
166         // load the rest of the data data and initialize the data members
167         reader.read(sprepBytes,mappingData);
168                                    
169         sprepTrie = new CharTrie(new ByteArrayInputStream  (sprepBytes), null);
170               
171         // get the data format version                           
172         formatVersion = reader.getDataFormatVersion();
173  
174         // get the options
175         doNFKC            = ((indexes[OPTIONS] & NORMALIZATION_ON) > 0);
176         checkBiDi         = ((indexes[OPTIONS] & CHECK_BIDI_ON) > 0);
177         sprepUniVer   = getVersionInfo(reader.getUnicodeVersion());
178         normCorrVer   = getVersionInfo(indexes[NORM_CORRECTNS_LAST_UNI_VERSION]);
179         VersionInfo normUniVer = Normalizer.getUnicodeVersion();
180         if(normUniVer.compareTo(sprepUniVer) < 0 && /* the Unicode version of SPREP file must be less than the Unicode Vesion of the normalization data */
181            normUniVer.compareTo(normCorrVer) < 0 && /* the Unicode version of the NormalizationCorrections.txt file should be less than the Unicode Vesion of the normalization data */
182            ((indexes[OPTIONS] & NORMALIZATION_ON) > 0) /* normalization turned on*/
183            ){
184             throw new IOException  ("Normalization Correction version not supported");
185         }
186         b.close();
187         
188         if(checkBiDi) {
189             bdp=UBiDiProps.getSingleton();
190         }
191     }
192  
193     private static final class Values{
194         boolean isIndex;
195         int value;
196         int type;
197         public void reset(){
198             isIndex = false;
199             value = 0;
200             type = -1;
201         }
202     }
203 
204     private static final void getValues(char trieWord,Values values){
205         values.reset();
206         if(trieWord == 0){
207             /* 
208              * Initial value stored in the mapping table 
209              * just return TYPE_LIMIT .. so that
210              * the source codepoint is copied to the destination
211              */
212             values.type = TYPE_LIMIT;
213         }else if(trieWord >= TYPE_THRESHOLD){
214             values.type = (trieWord - TYPE_THRESHOLD);
215         }else{
216             /* get the type */
217             values.type = MAP;
218             /* ascertain if the value is index or delta */
219             if((trieWord & 0x02)>0){
220                 values.isIndex = true;
221                 values.value = trieWord  >> 2; //mask off the lower 2 bits and shift
222 
223             }else{
224                 values.isIndex = false;
225                 values.value = ((int)(trieWord<<16))>>16;
226                 values.value =  (values.value >> 2);
227 
228             }
229  
230             if((trieWord>>2) == MAX_INDEX_VALUE){
231                 values.type = DELETE;
232                 values.isIndex = false;
233                 values.value = 0;
234             }
235         }
236     }
237 
238 
239 
240     private StringBuffer   map( UCharacterIterator iter, int options)
241                             throws StringPrepParseException{
242     
243         Values val = new Values();
244         char result = 0;
245         int ch  = UCharacterIterator.DONE;
246         StringBuffer   dest = new StringBuffer  ();
247         boolean allowUnassigned = ((options & ALLOW_UNASSIGNED)>0);
248         
249         while((ch=iter.nextCodePoint())!= UCharacterIterator.DONE){
250             
251             result = getCodePointValue(ch);
252             getValues(result,val);
253 
254             // check if the source codepoint is unassigned
255             if(val.type == UNASSIGNED && allowUnassigned == false){
256                  throw new StringPrepParseException("An unassigned code point was found in the input",
257                                           StringPrepParseException.UNASSIGNED_ERROR,
258                                           iter.getText(),iter.getIndex());    
259             }else if((val.type == MAP)){
260                 int index, length;
261 
262                 if(val.isIndex){
263                     index = val.value;
264                     if(index >= indexes[ONE_UCHAR_MAPPING_INDEX_START] &&
265                              index < indexes[TWO_UCHARS_MAPPING_INDEX_START]){
266                         length = 1;
267                     }else if(index >= indexes[TWO_UCHARS_MAPPING_INDEX_START] &&
268                              index < indexes[THREE_UCHARS_MAPPING_INDEX_START]){
269                         length = 2;
270                     }else if(index >= indexes[THREE_UCHARS_MAPPING_INDEX_START] &&
271                              index < indexes[FOUR_UCHARS_MAPPING_INDEX_START]){
272                         length = 3;
273                     }else{
274                         length = mappingData[index++];
275                     }
276                     /* copy mapping to destination */
277                     dest.append(mappingData,index,length);                    
278                     continue;
279                     
280                 }else{
281                     ch -= val.value;
282                 }
283             }else if(val.type == DELETE){
284                 // just consume the codepoint and contine
285                 continue;
286             }
287             //copy the source into destination
288             UTF16.append(dest,ch);
289         }
290         
291         return dest;
292     }
293 
294 
295     private StringBuffer   normalize(StringBuffer   src){
296         /*
297          * Option UNORM_BEFORE_PRI_29:
298          *
299          * IDNA as interpreted by IETF members (see unicode mailing list 2004H1)
300          * requires strict adherence to Unicode 3.2 normalization,
301          * including buggy composition from before fixing Public Review Issue #29.
302          * Note that this results in some valid but nonsensical text to be
303          * either corrupted or rejected, depending on the text.
304          * See http://www.unicode.org/review/resolved-pri.html#pri29
305          * See unorm.cpp and cnormtst.c
306          */
307         return new StringBuffer  (
308             Normalizer.normalize(
309                 src.toString(),
310                 Normalizer.NFKC,
311                 Normalizer.UNICODE_3_2|NormalizerImpl.BEFORE_PRI_29));
312     }
313     /*
314     boolean isLabelSeparator(int ch){
315         int result = getCodePointValue(ch);
316         if( (result & 0x07)  == LABEL_SEPARATOR){
317             return true;
318         }
319         return false;
320     }
321     */
322      /*
323        1) Map -- For each character in the input, check if it has a mapping
324           and, if so, replace it with its mapping.  
325 
326        2) Normalize -- Possibly normalize the result of step 1 using Unicode
327           normalization. 
328 
329        3) Prohibit -- Check for any characters that are not allowed in the
330           output.  If any are found, return an error.  
331 
332        4) Check bidi -- Possibly check for right-to-left characters, and if
333           any are found, make sure that the whole string satisfies the
334           requirements for bidirectional strings.  If the string does not
335           satisfy the requirements for bidirectional strings, return an
336           error.  
337           [Unicode3.2] defines several bidirectional categories; each character
338            has one bidirectional category assigned to it.  For the purposes of
339            the requirements below, an "RandALCat character" is a character that
340            has Unicode bidirectional categories "R" or "AL"; an "LCat character"
341            is a character that has Unicode bidirectional category "L".  Note
342 
343 
344            that there are many characters which fall in neither of the above
345            definitions; Latin digits (<U+0030> through <U+0039>) are examples of
346            this because they have bidirectional category "EN".
347 
348            In any profile that specifies bidirectional character handling, all
349            three of the following requirements MUST be met:
350 
351            1) The characters in section 5.8 MUST be prohibited.
352 
353            2) If a string contains any RandALCat character, the string MUST NOT
354               contain any LCat character.
355 
356            3) If a string contains any RandALCat character, a RandALCat
357               character MUST be the first character of the string, and a
358               RandALCat character MUST be the last character of the string.
359     */
360     /**
361      * Prepare the input buffer for use in applications with the given profile. This operation maps, normalizes(NFKC),
362      * checks for prohited and BiDi characters in the order defined by RFC 3454
363      * depending on the options specified in the profile.
364      *
365      * @param src           A UCharacterIterator object containing the source string
366      * @param options       A bit set of options:
367      *
368      *  - StringPrep.NONE               Prohibit processing of unassigned code points in the input
369      *
370      *  - StringPrep.ALLOW_UNASSIGNED   Treat the unassigned code points are in the input 
371      *                                  as normal Unicode code points.
372      *
373      * @return StringBuffer A StringBuffer containing the output
374      * @throws ParseException
375      * @stable ICU 2.8
376      */
377     public StringBuffer   prepare(UCharacterIterator src, int options)
378                         throws StringPrepParseException{
379                                               
380         // map 
381         StringBuffer   mapOut = map(src,options);
382         StringBuffer   normOut = mapOut;// initialize 
383         
384         if(doNFKC){
385             // normalize 
386             normOut = normalize(mapOut);
387         }
388 
389         int ch;
390         char result;
391         UCharacterIterator iter = UCharacterIterator.getInstance(normOut);
392         Values val = new Values(); 
393         int direction=UCharacterDirection.CHAR_DIRECTION_COUNT,
394             firstCharDir=UCharacterDirection.CHAR_DIRECTION_COUNT;    
395         int rtlPos=-1, ltrPos=-1;
396         boolean rightToLeft=false, leftToRight=false;
397            
398         while((ch=iter.nextCodePoint())!= UCharacterIterator.DONE){
399             result = getCodePointValue(ch);
400             getValues(result,val);
401 
402             if(val.type == PROHIBITED ){
403                 throw new StringPrepParseException("A prohibited code point was found in the input",
404                                          StringPrepParseException.PROHIBITED_ERROR,iter.getText(),val.value);
405             }
406 
407             if(checkBiDi) {
408                 direction = bdp.getClass(ch);
409                 if(firstCharDir == UCharacterDirection.CHAR_DIRECTION_COUNT){
410                     firstCharDir = direction;
411                 }
412                 if(direction == UCharacterDirection.LEFT_TO_RIGHT){
413                     leftToRight = true;
414                     ltrPos = iter.getIndex()-1;
415                 }
416                 if(direction == UCharacterDirection.RIGHT_TO_LEFT || direction == UCharacterDirection.RIGHT_TO_LEFT_ARABIC){
417                     rightToLeft = true;
418                     rtlPos = iter.getIndex()-1;
419                 }
420             }
421         }           
422         if(checkBiDi == true){
423             // satisfy 2
424             if( leftToRight == true && rightToLeft == true){
425                 throw new StringPrepParseException("The input does not conform to the rules for BiDi code points.",
426                                          StringPrepParseException.CHECK_BIDI_ERROR,iter.getText(),
427                                          (rtlPos>ltrPos) ? rtlPos : ltrPos);
428              }
429     
430             //satisfy 3
431             if( rightToLeft == true && 
432                 !((firstCharDir == UCharacterDirection.RIGHT_TO_LEFT || firstCharDir == UCharacterDirection.RIGHT_TO_LEFT_ARABIC) &&
433                 (direction == UCharacterDirection.RIGHT_TO_LEFT || direction == UCharacterDirection.RIGHT_TO_LEFT_ARABIC))
434               ){
435                 throw new StringPrepParseException("The input does not conform to the rules for BiDi code points.",
436                                          StringPrepParseException.CHECK_BIDI_ERROR,iter.getText(),
437                                          (rtlPos>ltrPos) ? rtlPos : ltrPos);
438             }
439         }
440         return normOut;
441 
442       }
443 }
444
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags