1 package fri.patterns.interpreter.parsergenerator.lexer; 2 3 import fri.patterns.interpreter.parsergenerator.Token; 4 5 41 42 public abstract class StandardLexerRules 43 { 44 81 public static String [][] rulesForIdentifier(String id) { 82 if (id.equals("char")) 84 return getUnicodeCharRules(); 85 if (id.equals("newline")) 86 return getNewlineRules(); 87 if (id.equals("newlines")) 88 return getNewlinesRules(); 89 if (id.equals("space")) 90 return getSpaceRules(); 91 if (id.equals("spaces")) 92 return getSpacesRules(); 93 if (id.equals("whitespace")) 94 return getWhitespaceRules(); 95 if (id.equals("whitespaces")) 96 return getWhitespacesRules(); 97 if (id.equals("letter")) 98 return getUnicodeLetterRules(); 99 if (id.equals("digit")) 100 return getUnicodeDigitRules(); 101 if (id.equals("digits")) 102 return getUnicodeDigitsRules(); 103 if (id.equals("hexdigit")) 104 return getHexDigitRules(); 105 if (id.equals("hexdigits")) 106 return getHexDigitsRules(); 107 if (id.equals("octdigits")) 108 return getOctDigitsRules(); 109 if (id.equals("bindigits")) 110 return getBinDigitsRules(); 111 if (id.equals("number")) 112 return getNumberRules(); 113 if (id.equals("integer")) 114 return getIntegerRules(); 115 if (id.equals("float")) 116 return getFloatRules(); 117 if (id.equals("identifier")) 118 return getUnicodeIdentifierRules(); 119 if (id.equals("stringdef")) 120 return getUnicodeStringdefRules(); 121 if (id.equals("chardef")) 122 return getUnicodeChardefRules(); 123 if (id.equals("bnf_chardef")) 124 return getUnicodeBNFChardefRules(); 125 if (id.equals("ruleref")) 126 return getRulerefRules(); 127 if (id.equals("quantifier")) 128 return getQuantifierRules(); 129 if (id.equals("comment")) 130 return getCommentRules(); 131 if (id.equals("cstylecomment")) 132 return getCStyleCommentRules(); 133 if (id.equals("shellstylecomment")) 134 return getShellStyleCommentRules(); 135 if (id.equals("xmlchar")) 136 return getUnicodeXmlCharRules(); 137 if (id.equals("combiningchar")) 138 return getUnicodeCombiningCharRules(); 139 if (id.equals("extenderchar")) 140 return getUnicodeExtenderCharRules(); 141 return null; 142 } 143 144 145 150 public static final String [][] getCustomOneLineCommentRules(String nonterminalName, String startChar) { 151 String [][] sarr0 = getUnicodeCharRules(); 152 String [][] sarr1 = getNewlineRules(); 153 String [][] sarr2 = getSomeRules(290, 296); 154 String [] customRule = new String [sarr2[0].length]; 155 System.arraycopy(sarr2[0], 0, customRule, 0, customRule.length); 156 customRule[0] = nonterminalName; 157 customRule[1] = "\""+startChar+"\""; sarr2[0] = customRule; 159 return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 }); 160 } 161 162 168 public static final String [][] getCustomMultiLineCommentRules(String nonterminalName, String startSeq, String endSeq) { 169 String [][] sarr0 = getUnicodeCharRules(); 170 String [][] sarr1 = getNewlineRules(); 171 String [][] customRules = new String [6][]; 172 customRules[0] = new String [] { nonterminalName, "\""+startSeq+"\"", "char_minus_star_slash_list_opt", "\""+endSeq+"\"" }; 173 customRules[1] = new String [] { "char_minus_"+nonterminalName, "char", Token.BUTNOT, "\""+endSeq+"\"" }; 174 customRules[2] = new String [] { "char_minus_"+nonterminalName+"_list", "char_minus_"+nonterminalName+"_list", "char_minus_"+nonterminalName }; 175 customRules[3] = new String [] { "char_minus_"+nonterminalName+"_list", "char_minus_"+nonterminalName }; 176 customRules[4] = new String [] { "char_minus_"+nonterminalName+"_list_opt", "char_minus_"+nonterminalName+"_list" }; 177 customRules[5] = new String [] { "char_minus_"+nonterminalName+"_list_opt" }; 178 return catenizeRules(new String [][][] { sarr0, sarr1, customRules }); 179 } 180 181 182 183 public static final String [][] getUnicodeCharRules() { 184 return getSomeRules(21, 22); 185 } 186 187 188 public static final String [][] getNewlineRules() { 189 return getSomeRules(16, 21); 190 } 191 192 193 public static final String [][] getNewlinesRules() { 194 String [][] sarr0 = getNewlineRules(); 195 String [][] sarr1 = getSomeRules(0, 2, newlinesRules); 196 return catenizeRules(new String [][][] { sarr0, sarr1 }); 197 } 198 199 200 public static final String [][] getSpaceRules() { 201 return getSomeRules(13, 16); 202 } 203 204 205 public static final String [][] getSpacesRules() { 206 String [][] sarr0 = getSpaceRules(); 207 String [][] sarr1 = getSomeRules(242, 244); 208 return catenizeRules(new String [][][] { sarr0, sarr1 }); 209 } 210 211 212 public static final String [][] getWhitespaceRules() { 213 String [][] sarr0 = getSpaceRules(); 214 String [][] sarr1 = getNewlineRules(); 215 String [][] sarr2 = getSomeRules(0, 2, whitespaceRules); 216 return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 }); 217 } 218 219 220 public static final String [][] getWhitespacesRules() { 221 String [][] sarr0 = getSpacesRules(); 222 String [][] sarr1 = getNewlinesRules(); 223 String [][] sarr2 = getSomeRules(0, 4, whitespaceRules); 224 return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 }); 225 } 226 227 228 public static final String [][] getHexDigitRules() { 229 return getSomeRules(10, 13); 230 } 231 232 233 public static final String [][] getHexDigitsRules() { 234 String [][] sarr0 = getHexDigitRules(); 235 String [][] sarr1 = getSomeRules(246, 248); return catenizeRules(new String [][][] { sarr0, sarr1 }); 237 } 238 239 240 public static final String [][] getUnicodeLetterRules() { 241 return getSomeRules(37, 242); 242 } 243 244 245 public static final String [][] getUnicodeDigitRules() { 246 return getSomeRules(22, 37); 247 } 248 249 250 public static final String [][] getUnicodeDigitsRules() { 251 String [][] sarr0 = getUnicodeDigitRules(); 252 String [][] sarr1 = getSomeRules(244, 246); return catenizeRules(new String [][][] { sarr0, sarr1 }); 254 } 255 256 257 public static final String [][] getUnicodeIdentifierRules() { 258 String [][] sarr0 = getUnicodeDigitRules(); 259 String [][] sarr1 = getUnicodeLetterRules(); 260 String [][] sarr2 = getSomeRules(259, 268); 261 return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 }); 262 } 263 264 265 public static final String [][] getUnicodeChardefRules() { 266 String [][] sarr0 = getUnicodeCharRules(); 267 String [][] sarr1 = getSomeRules(0, 1, digitRules); String [][] sarr2 = getSomeRules(0, 2, chardefRules); 269 String [][] sarr3 = getSomeRules(248, 249); String [][] sarr4 = getSomeRules(251, 258); return catenizeRules(new String [][][] { sarr0, sarr1, sarr2, sarr3, sarr4 }); 272 } 273 274 275 public static final String [][] getUnicodeBNFChardefRules() { 276 String [][] sarr0 = getUnicodeCharRules(); 277 String [][] sarr1 = getHexDigitsRules(); 278 String [][] sarr2 = getUnicodeDigitsRules(); 279 String [][] sarr3 = getSomeRules(248, 259); 280 return catenizeRules(new String [][][] { sarr0, sarr1, sarr2, sarr3 }); 281 } 282 283 284 public static final String [][] getUnicodeStringdefRules() { 285 String [][] sarr0 = getUnicodeCharRules(); 286 String [][] sarr1 = getSomeRules(268, 284); 287 return catenizeRules(new String [][][] { sarr0, sarr1 }); 288 } 289 290 291 public static final String [][] getRulerefRules() { 292 String [][] sarr0 = getUnicodeIdentifierRules(); 293 String [][] sarr1 = getSomeRules(297, 298); 294 return catenizeRules(new String [][][] { sarr0, sarr1 }); 295 } 296 297 298 public static final String [][] getQuantifierRules() { 299 return getSomeRules(7, 10); 300 } 301 302 303 public static final String [][] getCommentRules() { 304 String [][] sarr0 = getCStyleCommentRules(); 305 String [][] sarr1 = getSomeRules(296, 297); 306 String [][] sarr2 = getSomeRules(299, 301); 307 return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 }); 308 } 309 310 311 public static final String [][] getCStyleCommentRules() { 312 String [][] sarr0 = getUnicodeCharRules(); 313 String [][] sarr1 = getNewlineRules(); 314 String [][] sarr2 = getSomeRules(284, 296); 315 return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 }); 316 } 317 318 319 public static final String [][] getShellStyleCommentRules() { 320 String [][] sarr0 = getUnicodeCharRules(); 321 String [][] sarr1 = getNewlineRules(); 322 String [][] sarr2 = getSomeRules(291, 297); 323 return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 }); 324 } 325 326 327 public static final String [][] getUnicodeXmlCharRules() { 328 return xmlCharRules; 329 } 330 331 332 public static final String [][] getUnicodeCombiningCharRules() { 333 String [][] sarr = getSomeRules(0, 95, xmlCombinigAndExtenderRules); 334 return sarr; 335 } 336 337 338 public static final String [][] getUnicodeExtenderCharRules() { 339 String [][] sarr = getSomeRules(95, 106, xmlCombinigAndExtenderRules); 340 return sarr; 341 } 342 343 344 public static final String [][] getOctDigitsRules() { 345 String [][] sarr = getSomeRules(0, 3, digitRules); 346 return sarr; 347 } 348 349 350 public static final String [][] getBinDigitsRules() { 351 String [][] sarr = getSomeRules(3, 6, digitRules); 352 return sarr; 353 } 354 355 356 public static final String [][] getNumberRules() { 357 String [][] sarr2 = getIntegerRules(); 358 String [][] sarr1 = getFloatRules(); 359 String [][] sarr0 = getSomeRules(0, 2, numberRules); 360 return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 }); 361 } 362 363 364 public static final String [][] getIntegerRules() { 365 String [][] sarr2 = getHexDigitsRules(); 366 String [][] sarr1 = getUnicodeDigitsRules(); 367 String [][] sarr0 = getSomeRules(19, 25, numberRules); 368 return catenizeRules(new String [][][] { sarr0, sarr1, sarr2 }); 369 } 370 371 372 public static final String [][] getFloatRules() { 373 String [][] sarr1 = getUnicodeDigitsRules(); 374 String [][] sarr0 = getSomeRules(2, 19, numberRules); 375 return catenizeRules(new String [][][] { sarr0, sarr1 }); 376 } 377 378 379 private static final String [][] getSomeRules(int startIncl, int endExcl) { 380 return getSomeRules(startIncl, endExcl, lexerSyntax); 381 } 382 383 private static final String [][] getSomeRules(int startIncl, int endExcl, String [][] rules) { 384 String [][] sarr = new String [endExcl - startIncl] []; 385 int j = 0; 386 for (int i = startIncl; i < endExcl; i++, j++) 387 sarr[j] = rules[i]; 388 return sarr; 389 } 390 391 392 393 public static void printRules(String [][] syntax) { 394 for (int i = 0; i < syntax.length; i++) { 395 for (int j = 0; j < syntax[i].length; j++) 396 System.out.print(j == 1 ? " ::= "+syntax[i][j]+" " : syntax[i][j]+" "); 397 System.out.println(); 398 } 399 } 400 401 402 403 public static final String [][] catenizeRules(String [][][] arrays) { 404 int len = 0; 405 for (int i = 0; i < arrays.length; i++) 406 len += arrays[i].length; 407 408 String [][] sarr = new String [len][]; 409 410 int k = 0; 411 for (int i = 0; i < arrays.length; i++) { 412 for (int j = 0; j < arrays[i].length; j++) { 413 sarr[k] = arrays[i][j]; 414 k++; 415 } 416 } 417 418 return sarr; 419 } 420 421 422 423 424 425 public static final String [][] lexerSyntax = { 426 427 429 { Token.TOKEN, "identifier" }, 430 { Token.TOKEN, "bnf_chardef" }, 431 { Token.TOKEN, "stringdef" }, 432 { Token.TOKEN, "quantifier" }, 434 { Token.IGNORED, "spaces" }, 435 { Token.IGNORED, "newline" }, 436 { Token.IGNORED, "comment" }, 437 438 { "quantifier", "'*'" }, 439 { "quantifier", "'+'" }, 440 { "quantifier", "'?'" }, 441 442 444 { "hexdigit", "'0'", Token.UPTO, "'9'" }, 445 { "hexdigit", "'A'", Token.UPTO, "'F'" }, 446 { "hexdigit", "'a'", Token.UPTO, "'f'" }, 447 448 450 { "space", "0x20" }, 451 { "space", "0x9" }, 452 { "space", "0xC" }, 454 { "cr", "'\\r'" }, { "nl", "'\\n'" }, { "newline", "cr", "nl" }, 457 { "newline", "cr" }, 458 { "newline", "nl" }, 459 460 462 { "char", "0x0", Token.UPTO, "0xFFFF" }, 463 464 { "digit", "0x0030", Token.UPTO, "0x0039" }, 465 { "digit", "0x0660", Token.UPTO, "0x0669" }, 466 { "digit", "0x06F0", Token.UPTO, "0x06F9" }, 467 { "digit", "0x0966", Token.UPTO, "0x096F" }, 468 { "digit", "0x09E6", Token.UPTO, "0x09EF" }, 469 { "digit", "0x0A66", Token.UPTO, "0x0A6F" }, 470 { "digit", "0x0AE6", Token.UPTO, "0x0AEF" }, 471 { "digit", "0x0B66", Token.UPTO, "0x0B6F" }, 472 { "digit", "0x0BE7", Token.UPTO, "0x0BEF" }, 473 { "digit", "0x0C66", Token.UPTO, "0x0C6F" }, 474 { "digit", "0x0CE6", Token.UPTO, "0x0CEF" }, 475 { "digit", "0x0D66", Token.UPTO, "0x0D6F" }, 476 { "digit", "0x0E50", Token.UPTO, "0x0E59" }, 477 { "digit", "0x0ED0", Token.UPTO, "0x0ED9" }, 478 { "digit", "0x0F20", Token.UPTO, "0x0F29" }, 479 480 { "letter", "0x0041", Token.UPTO, "0x005A" }, { "letter", "0x0061", Token.UPTO, "0x007A" }, 482 { "letter", "0x00C0", Token.UPTO, "0x00D6" }, 483 { "letter", "0x00D8", Token.UPTO, "0x00F6" }, 484 { "letter", "0x00F8", Token.UPTO, "0x00FF" }, 485 { "letter", "0x0100", Token.UPTO, "0x0131" }, 486 { "letter", "0x0134", Token.UPTO, "0x013E" }, 487 { "letter", "0x0141", Token.UPTO, "0x0148" }, 488 { "letter", "0x014A", Token.UPTO, "0x017E" }, 489 { "letter", "0x0180", Token.UPTO, "0x01C3" }, 490 { "letter", "0x01CD", Token.UPTO, "0x01F0" }, 491 { "letter", "0x01F4", Token.UPTO, "0x01F5" }, 492 { "letter", "0x01FA", Token.UPTO, "0x0217" }, 493 { "letter", "0x0250", Token.UPTO, "0x02A8" }, 494 { "letter", "0x02BB", Token.UPTO, "0x02C1" }, 495 { "letter", "0x0386" }, 496 { "letter", "0x0388", Token.UPTO, "0x038A" }, 497 { "letter", "0x038C" }, 498 { "letter", "0x038E", Token.UPTO, "0x03A1" }, 499 { "letter", "0x03A3", Token.UPTO, "0x03CE" }, 500 { "letter", "0x03D0", Token.UPTO, "0x03D6" }, 501 { "letter", "0x03DA" }, 502 { "letter", "0x03DC" }, 503 { "letter", "0x03DE" }, 504 { "letter", "0x03E0" }, 505 { "letter", "0x03E2", Token.UPTO, "0x03F3" }, 506 { "letter", "0x0401", Token.UPTO, "0x040C" }, 507 { "letter", "0x040E", Token.UPTO, "0x044F" }, 508 { "letter", "0x0451", Token.UPTO, "0x045C" }, 509 { "letter", "0x045E", Token.UPTO, "0x0481" }, 510 { "letter", "0x0490", Token.UPTO, "0x04C4" }, 511 { "letter", "0x04C7", Token.UPTO, "0x04C8" }, 512 { "letter", "0x04CB", Token.UPTO, "0x04CC" }, 513 { "letter", "0x04D0", Token.UPTO, "0x04EB" }, 514 { "letter", "0x04EE", Token.UPTO, "0x04F5" }, 515 { "letter", "0x04F8", Token.UPTO, "0x04F9" }, 516 { "letter", "0x0531", Token.UPTO, "0x0556" }, 517 { "letter", "0x0559" }, 518 { "letter", "0x0561", Token.UPTO, "0x0586" }, 519 { "letter", "0x05D0", Token.UPTO, "0x05EA" }, 520 { "letter", "0x05F0", Token.UPTO, "0x05F2" }, 521 { "letter", "0x0621", Token.UPTO, "0x063A" }, 522 { "letter", "0x0641", Token.UPTO, "0x064A" }, 523 { "letter", "0x0671", Token.UPTO, "0x06B7" }, 524 { "letter", "0x06BA", Token.UPTO, "0x06BE" }, 525 { "letter", "0x06C0", Token.UPTO, "0x06CE" }, 526 { "letter", "0x06D0", Token.UPTO, "0x06D3" }, 527 { "letter", "0x06D5" }, 528 { "letter", "0x06E5", Token.UPTO, "0x06E6" }, 529 { "letter", "0x0905", Token.UPTO, "0x0939" }, 530 { "letter", "0x093D" }, 531 { "letter", "0x0958", Token.UPTO, "0x0961" }, 532 { "letter", "0x0985", Token.UPTO, "0x098C" }, 533 { "letter", "0x098F", Token.UPTO, "0x0990" }, 534 { "letter", "0x0993", Token.UPTO, "0x09A8" }, 535 { "letter", "0x09AA", Token.UPTO, "0x09B0" }, 536 { "letter", "0x09B2" }, 537 { "letter", "0x09B6", Token.UPTO, "0x09B9" }, 538 { "letter", "0x09DC", Token.UPTO, "0x09DD" }, 539 { "letter", "0x09DF", Token.UPTO, "0x09E1" }, 540 { "letter", "0x09F0", Token.UPTO, "0x09F1" }, 541 { "letter", "0x0A05", Token.UPTO, "0x0A0A" }, 542 { "letter", "0x0A0F", Token.UPTO, "0x0A10" }, 543 { "letter", "0x0A13", Token.UPTO, "0x0A28" }, 544 { "letter", "0x0A2A", Token.UPTO, "0x0A30" }, 545 { "letter", "0x0A32", Token.UPTO, "0x0A33" }, 546 { "letter", "0x0A35", Token.UPTO, "0x0A36" }, 547 { "letter", "0x0A38", Token.UPTO, "0x0A39" }, 548 { "letter", "0x0A59", Token.UPTO, "0x0A5C" }, 549 { "letter", "0x0A5E" }, 550 { "letter", "0x0A72", Token.UPTO, "0x0A74" }, 551 { "letter", "0x0A85", Token.UPTO, "0x0A8B" }, 552 { "letter", "0x0A8D" }, 553 { "letter", "0x0A8F", Token.UPTO, "0x0A91" }, 554 { "letter", "0x0A93", Token.UPTO, "0x0AA8" }, 555 { "letter", "0x0AAA", Token.UPTO, "0x0AB0" }, 556 { "letter", "0x0AB2", Token.UPTO, "0x0AB3" }, 557 { "letter", "0x0AB5", Token.UPTO, "0x0AB9" }, 558 { "letter", "0x0ABD" }, 559 { "letter", "0x0AE0" }, 560 { "letter", "0x0B05", Token.UPTO, "0x0B0C" }, 561 { "letter", "0x0B0F", Token.UPTO, "0x0B10" }, 562 { "letter", "0x0B13", Token.UPTO, "0x0B28" }, 563 { "letter", "0x0B2A", Token.UPTO, "0x0B30" }, 564 { "letter", "0x0B32", Token.UPTO, "0x0B33" }, 565 { "letter", "0x0B36", Token.UPTO, "0x0B39" }, 566 { "letter", "0x0B3D" }, 567 { "letter", "0x0B5C", Token.UPTO, "0x0B5D" }, 568 { "letter", "0x0B5F", Token.UPTO, "0x0B61" }, 569 { "letter", "0x0B85", Token.UPTO, "0x0B8A" }, 570 { "letter", "0x0B8E", Token.UPTO, "0x0B90" }, 571 { "letter", "0x0B92", Token.UPTO, "0x0B95" }, 572 { "letter", "0x0B99", Token.UPTO, "0x0B9A" }, 573 { "letter", "0x0B9C" }, 574 { "letter", "0x0B9E", Token.UPTO, "0x0B9F" }, 575 { "letter", "0x0BA3", Token.UPTO, "0x0BA4" }, 576 { "letter", "0x0BA8", Token.UPTO, "0x0BAA" }, 577 { "letter", "0x0BAE", Token.UPTO, "0x0BB5" }, 578 { "letter", "0x0BB7", Token.UPTO, "0x0BB9" }, 579 { "letter", "0x0C05", Token.UPTO, "0x0C0C" }, 580 { "letter", "0x0C0E", Token.UPTO, "0x0C10" }, 581 { "letter", "0x0C12", Token.UPTO, "0x0C28" }, 582 { "letter", "0x0C2A", Token.UPTO, "0x0C33" }, 583 { "letter", "0x0C35", Token.UPTO, "0x0C39" }, 584 { "letter", "0x0C60", Token.UPTO, "0x0C61" }, 585 { "letter", "0x0C85", Token.UPTO, "0x0C8C" }, 586 { "letter", "0x0C8E", Token.UPTO, "0x0C90" }, 587 { "letter", "0x0C92", Token.UPTO, "0x0CA8" }, 588 { "letter", "0x0CAA", Token.UPTO, "0x0CB3" }, 589 { "letter", "0x0CB5", Token.UPTO, "0x0CB9" }, 590 { "letter", "0x0CDE" }, 591 { "letter", "0x0CE0", Token.UPTO, "0x0CE1" }, 592 { "letter", "0x0D05", Token.UPTO, "0x0D0C" }, 593 { "letter", "0x0D0E", Token.UPTO, "0x0D10" }, 594 { "letter", "0x0D12", Token.UPTO, "0x0D28" }, 595 { "letter", "0x0D2A", Token.UPTO, "0x0D39" }, 596 { "letter", "0x0D60", Token.UPTO, "0x0D61" }, 597 { "letter", "0x0E01", Token.UPTO, "0x0E2E" }, 598 { "letter", "0x0E30" }, 599 { "letter", "0x0E32", Token.UPTO, "0x0E33" }, 600 { "letter", "0x0E40", Token.UPTO, "0x0E45" }, 601 { "letter", "0x0E81", Token.UPTO, "0x0E82" }, 602 { "letter", "0x0E84" }, 603 { "letter", "0x0E87", Token.UPTO, "0x0E88" }, 604 { "letter", "0x0E8A" }, 605 { "letter", "0x0E8D" }, 606 { "letter", "0x0E94", Token.UPTO, "0x0E97" }, 607 { "letter", "0x0E99", Token.UPTO, "0x0E9F" }, 608 { "letter", "0x0EA1", Token.UPTO, "0x0EA3" }, 609 { "letter", "0x0EA5" }, 610 { "letter", "0x0EA7" }, 611 { "letter", "0x0EAA", Token.UPTO, "0x0EAB" }, 612 { "letter", "0x0EAD", Token.UPTO, "0x0EAE" }, 613 { "letter", "0x0EB0" }, 614 { "letter", "0x0EB2", Token.UPTO, "0x0EB3" }, 615 { "letter", "0x0EBD" }, 616 { "letter", "0x0EC0", Token.UPTO, "0x0EC4" }, 617 { "letter", "0x0F40", Token.UPTO, "0x0F47" }, 618 { "letter", "0x0F49", Token.UPTO, "0x0F69" }, 619 { "letter", "0x10A0", Token.UPTO, "0x10C5" }, 620 { "letter", "0x10D0", Token.UPTO, "0x10F6" }, 621 { "letter", "0x1100" }, 622 { "letter", "0x1102", Token.UPTO, "0x1103" }, 623 { "letter", "0x1105", Token.UPTO, "0x1107" }, 624 { "letter", "0x1109" }, 625 { "letter", "0x110B", Token.UPTO, "0x110C" }, 626 { "letter", "0x110E", Token.UPTO, "0x1112" }, 627 { "letter", "0x113C" }, 628 { "letter", "0x113E" }, 629 { "letter", "0x1140" }, 630 { "letter", "0x114C" }, 631 { "letter", "0x114E" }, 632 { "letter", "0x1150" }, 633 { "letter", "0x1154", Token.UPTO, "0x1155" }, 634 { "letter", "0x1159" }, 635 { "letter", "0x115F", Token.UPTO, "0x1161" }, 636 { "letter", "0x1163" }, 637 { "letter", "0x1165" }, 638 { "letter", "0x1167" }, 639 { "letter", "0x1169" }, 640 { "letter", "0x116D", Token.UPTO, "0x116E" }, 641 { "letter", "0x1172", Token.UPTO, "0x1173" }, 642 { "letter", "0x1175" }, 643 { "letter", "0x119E" }, 644 { "letter", "0x11A8" }, 645 { "letter", "0x11AB" }, 646 { "letter", "0x11AE", Token.UPTO, "0x11AF" }, 647 { "letter", "0x11B7", Token.UPTO, "0x11B8" }, 648 { "letter", "0x11BA" }, 649 { "letter", "0x11BC", Token.UPTO, "0x11C2" }, 650 { "letter", "0x11EB" }, 651 { "letter", "0x11F0" }, 652 { "letter", "0x11F9" }, 653 { "letter", "0x1E00", Token.UPTO, "0x1E9B" }, 654 { "letter", "0x1EA0", Token.UPTO, "0x1EF9" }, 655 { "letter", "0x1F00", Token.UPTO, "0x1F15" }, 656 { "letter", "0x1F18", Token.UPTO, "0x1F1D" }, 657 { "letter", "0x1F20", Token.UPTO, "0x1F45" }, 658 { "letter", "0x1F48", Token.UPTO, "0x1F4D" }, 659 { "letter", "0x1F50", Token.UPTO, "0x1F57" }, 660 { "letter", "0x1F59" }, 661 { "letter", "0x1F5B" }, 662 { "letter", "0x1F5D" }, 663 { "letter", "0x1F5F", Token.UPTO, "0x1F7D" }, 664 { "letter", "0x1F80", Token.UPTO, "0x1FB4" }, 665 { "letter", "0x1FB6", Token.UPTO, "0x1FBC" }, 666 { "letter", "0x1FBE" }, 667 { "letter", "0x1FC2", Token.UPTO, "0x1FC4" }, 668 { "letter", "0x1FC6", Token.UPTO, "0x1FCC" }, 669 { "letter", "0x1FD0", Token.UPTO, "0x1FD3" }, 670 { "letter", "0x1FD6", Token.UPTO, "0x1FDB" }, 671 { "letter", "0x1FE0", Token.UPTO, "0x1FEC" }, 672 { "letter", "0x1FF2", Token.UPTO, "0x1FF4" }, 673 { "letter", "0x1FF6", Token.UPTO, "0x1FFC" }, 674 { "letter", "0x2126" }, 675 { "letter", "0x212A", Token.UPTO, "0x212B" }, 676 { "letter", "0x212E" }, 677 { "letter", "0x2180", Token.UPTO, "0x2182" }, 678 { "letter", "0x3041", Token.UPTO, "0x3094" }, 679 { "letter", "0x30A1", Token.UPTO, "0x30FA" }, 680 { "letter", "0x3105", Token.UPTO, "0x312C" }, 681 { "letter", "0xAC00", Token.UPTO, "0xD7A3" }, 682 { "letter", "0x4E00", Token.UPTO, "0x9FA5" }, 684 { "letter", "0x3007" }, 685 { "letter", "0x3021", Token.UPTO, "0x3029" }, 686 687 689 { "spaces", "spaces", "space" }, 690 { "spaces", "space" }, 691 692 { "digits", "digits", "digit" }, 693 { "digits", "digit" }, 694 695 { "hexdigits", "hexdigits", "hexdigit" }, 696 { "hexdigits", "hexdigit" }, 697 698 { "bnf_chardef", "\"'\"", "char", "\"'\"" }, 699 { "bnf_chardef", "\"0x\"", "hexdigits" }, 700 { "bnf_chardef", "\"0X\"", "hexdigits" }, 701 { "bnf_chardef", "\"'\\''\"" }, { "bnf_chardef", "\"'\\n'\"" }, { "bnf_chardef", "\"'\\r'\"" }, { "bnf_chardef", "\"'\\t'\"" }, { "bnf_chardef", "\"'\\f'\"" }, { "bnf_chardef", "\"'\\b'\"" }, { "bnf_chardef", "\"'\\\\'\"" }, { "bnf_chardef", "digits" }, 710 711 { "identifier", "letter_or_uscore", "letter_or_digit_list_opt" }, 712 { "letter_or_uscore", "letter" }, 713 { "letter_or_uscore", "'_'" }, 714 { "letter_or_digit", "letter_or_uscore" }, 715 { "letter_or_digit", "digit" }, 716 { "letter_or_digit_list", "letter_or_digit_list", "letter_or_digit" }, 717 { "letter_or_digit_list", "letter_or_digit" }, 718 { "letter_or_digit_list_opt", "letter_or_digit_list" }, 719 { "letter_or_digit_list_opt" }, 720 721 { "stringdef", "'\"'", "stringpart_list_opt", "'\"'" }, 722 { "char_minus_doublequote_list", "char_minus_doublequote_list", "char_minus_doublequote" }, 723 { "char_minus_doublequote_list", "char_minus_doublequote" }, 724 { "char_minus_doublequote", "char", Token.BUTNOT, "'\"'", Token.BUTNOT, "'\\'" }, { "char_minus_doublequote_list_opt", "char_minus_doublequote_list" }, 726 { "char_minus_doublequote_list_opt" }, 727 { "backslash_char_list", "backslash_char_list", "backslash_char" }, 728 { "backslash_char_list", "backslash_char" }, 729 { "backslash_char", "'\\'", "char" }, { "backslash_char_list_opt", "backslash_char_list" }, 731 { "backslash_char_list_opt" }, 732 { "stringpart", "char_minus_doublequote_list_opt", "backslash_char_list_opt" }, 733 { "stringpart_list", "stringpart_list", "stringpart" }, 734 { "stringpart_list", "stringpart" }, 735 { "stringpart_list_opt", "stringpart_list" }, 736 { "stringpart_list_opt" }, 737 738 { "cstylecomment", "\"/*\"", "char_minus_star_slash_list_opt", "\"*/\"" }, 739 { "char_minus_star_slash", "char", Token.BUTNOT, "\"*/\"" }, 740 { "char_minus_star_slash_list", "char_minus_star_slash_list", "char_minus_star_slash" }, 741 { "char_minus_star_slash_list", "char_minus_star_slash" }, 742 { "char_minus_star_slash_list_opt", "char_minus_star_slash_list" }, 743 { "char_minus_star_slash_list_opt" }, 744 745 { "cstylecomment", "\"//\"", "char_minus_newline_list_opt" }, 746 { "char_minus_newline", "char", Token.BUTNOT, "newline" }, 747 { "char_minus_newline_list", "char_minus_newline_list", "char_minus_newline" }, 748 { "char_minus_newline_list", "char_minus_newline" }, 749 { "char_minus_newline_list_opt", "char_minus_newline_list" }, 750 { "char_minus_newline_list_opt" }, 751 { "shellstylecomment", "'#'", "char_minus_newline_list_opt" }, 752 753 { "ruleref", "'"+Token.COMMAND_QUOTE+"'", "identifier", "'"+Token.COMMAND_QUOTE+"'" }, 754 { Token.TOKEN, "ruleref" }, 755 756 { "comment", "cstylecomment" }, 757 { "comment", "shellstylecomment" }, 758 }; 759 760 761 public static final String [][] xmlCharRules = { 762 { "xmlchar", "0x9" }, 763 { "xmlchar", "0xA" }, 764 { "xmlchar", "0xD" }, 765 { "xmlchar", "0x20", Token.UPTO, "0xD7FF" }, 766 { "xmlchar", "0xE000", Token.UPTO, "0xFFFD" }, 767 { "xmlchar", "0x10000", Token.UPTO, "0x10FFFF" }, 768 }; 769 770 771 public static final String [][] xmlCombinigAndExtenderRules = { 772 { "combiningchar", "0x0300", Token.UPTO, "0x0345" }, 773 { "combiningchar", "0x0360", Token.UPTO, "0x0361" }, 774 { "combiningchar", "0x0483", Token.UPTO, "0x0486" }, 775 { "combiningchar", "0x0591", Token.UPTO, "0x05A1" }, 776 { "combiningchar", "0x05A3", Token.UPTO, "0x05B9" }, 777 { "combiningchar", "0x05BB", Token.UPTO, "0x05BD" }, 778 { "combiningchar", "0x05BF" }, 779 { "combiningchar", "0x05C1", Token.UPTO, "0x05C2" }, 780 { "combiningchar", "0x05C4" }, 781 { "combiningchar", "0x064B", Token.UPTO, "0x0652" }, 782 { "combiningchar", "0x0670" }, 783 { "combiningchar", "0x06D6", Token.UPTO, "0x06DC" }, 784 { "combiningchar", "0x06DD", Token.UPTO, "0x06DF" }, 785 { "combiningchar", "0x06E0", Token.UPTO, "0x06E4" }, 786 { "combiningchar", "0x06E7", Token.UPTO, "0x06E8" }, 787 { "combiningchar", "0x06EA", Token.UPTO, "0x06ED" }, 788 { "combiningchar", "0x0901", Token.UPTO, "0x0903" }, 789 { "combiningchar", "0x093C" }, 790 { "combiningchar", "0x093E", Token.UPTO, "0x094C" }, 791 { "combiningchar", "0x094D" }, 792 { "combiningchar", "0x0951", Token.UPTO, "0x0954" }, 793 { "combiningchar", "0x0962", Token.UPTO, "0x0963" }, 794 { "combiningchar", "0x0981", Token.UPTO, "0x0983" }, 795 { "combiningchar", "0x09BC" }, 796 { "combiningchar", "0x09BE" }, 797 { "combiningchar", "0x09BF" }, 798 { "combiningchar", "0x09C0", Token.UPTO, "0x09C4" }, 799 { "combiningchar", "0x09C7", Token.UPTO, "0x09C8" }, 800 { "combiningchar", "0x09CB", Token.UPTO, "0x09CD" }, 801 { "combiningchar", "0x09D7" }, 802 { "combiningchar", "0x09E2", Token.UPTO, "0x09E3" }, 803 { "combiningchar", "0x0A02" }, 804 { "combiningchar", "0x0A3C" }, 805 { "combiningchar", "0x0A3E" }, 806 { "combiningchar", "0x0A3F" }, 807 { "combiningchar", "0x0A40", Token.UPTO, "0x0A42" }, 808 { "combiningchar", "0x0A47", Token.UPTO, "0x0A48" }, 809 { "combiningchar", "0x0A4B", Token.UPTO, "0x0A4D" }, 810 { "combiningchar", "0x0A70", Token.UPTO, "0x0A71" }, 811 { "combiningchar", "0x0A81", Token.UPTO, "0x0A83" }, 812 { "combiningchar", "0x0ABC" }, 813 { "combiningchar", "0x0ABE", Token.UPTO, "0x0AC5" }, 814 { "combiningchar", "0x0AC7", Token.UPTO, "0x0AC9" }, 815 { "combiningchar", "0x0ACB", Token.UPTO, "0x0ACD" }, 816 { "combiningchar", "0x0B01", Token.UPTO, "0x0B03" }, 817 { "combiningchar", "0x0B3C" }, 818 { "combiningchar", "0x0B3E", Token.UPTO, "0x0B43" }, 819 { "combiningchar", "0x0B47", Token.UPTO, "0x0B48" }, 820 { "combiningchar", "0x0B4B", Token.UPTO, "0x0B4D" }, 821 { "combiningchar", "0x0B56", Token.UPTO, "0x0B57" }, 822 { "combiningchar", "0x0B82", Token.UPTO, "0x0B83" }, 823 { "combiningchar", "0x0BBE", Token.UPTO, "0x0BC2" }, 824 { "combiningchar", "0x0BC6", Token.UPTO, "0x0BC8" }, 825 { "combiningchar", "0x0BCA", Token.UPTO, "0x0BCD" }, 826 { "combiningchar", "0x0BD7" }, 827 { "combiningchar", "0x0C01", Token.UPTO, "0x0C03" }, 828 { "combiningchar", "0x0C3E", Token.UPTO, "0x0C44" }, 829 { "combiningchar", "0x0C46", Token.UPTO, "0x0C48" }, 830 { "combiningchar", "0x0C4A", Token.UPTO, "0x0C4D" }, 831 { "combiningchar", "0x0C55", Token.UPTO, "0x0C56" }, 832 { "combiningchar", "0x0C82", Token.UPTO, "0x0C83" }, 833 { "combiningchar", "0x0CBE", Token.UPTO, "0x0CC4" }, 834 { "combiningchar", "0x0CC6", Token.UPTO, "0x0CC8" }, 835 { "combiningchar", "0x0CCA", Token.UPTO, "0x0CCD" }, 836 { "combiningchar", "0x0CD5", Token.UPTO, "0x0CD6" }, 837 { "combiningchar", "0x0D02", Token.UPTO, "0x0D03" }, 838 { "combiningchar", "0x0D3E", Token.UPTO, "0x0D43" }, 839 { "combiningchar", "0x0D46", Token.UPTO, "0x0D48" }, 840 { "combiningchar", "0x0D4A", Token.UPTO, "0x0D4D" }, 841 { "combiningchar", "0x0D57" }, 842 { "combiningchar", "0x0E31" }, 843 { "combiningchar", "0x0E34", Token.UPTO, "0x0E3A" }, 844 { "combiningchar", "0x0E47", Token.UPTO, "0x0E4E" }, 845 { "combiningchar", "0x0EB1" }, 846 { "combiningchar", "0x0EB4", Token.UPTO, "0x0EB9" }, 847 { "combiningchar", "0x0EBB", Token.UPTO, "0x0EBC" }, 848 { "combiningchar", "0x0EC8", Token.UPTO, "0x0ECD" }, 849 { "combiningchar", "0x0F18", Token.UPTO, "0x0F19" }, 850 { "combiningchar", "0x0F35" }, 851 { "combiningchar", "0x0F37" }, 852 { "combiningchar", "0x0F39" }, 853 { "combiningchar", "0x0F3E" }, 854 { "combiningchar", "0x0F3F" }, 855 { "combiningchar", "0x0F71", Token.UPTO, "0x0F84" }, 856 { "combiningchar", "0x0F86", Token.UPTO, "0x0F8B" }, 857 { "combiningchar", "0x0F90", Token.UPTO, "0x0F95" }, 858 { "combiningchar", "0x0F97" }, 859 { "combiningchar", "0x0F99", Token.UPTO, "0x0FAD" }, 860 { "combiningchar", "0x0FB1", Token.UPTO, "0x0FB7" }, 861 { "combiningchar", "0x0FB9" }, 862 { "combiningchar", "0x20D0", Token.UPTO, "0x20DC" }, 863 { "combiningchar", "0x20E1" }, 864 { "combiningchar", "0x302A", Token.UPTO, "0x302F" }, 865 { "combiningchar", "0x3099" }, 866 { "combiningchar", "0x309A" }, 867 868 { "extenderchar", "0x00B7" }, 869 { "extenderchar", "0x02D0" }, 870 { "extenderchar", "0x02D1" }, 871 { "extenderchar", "0x0387" }, 872 { "extenderchar", "0x0640" }, 873 { "extenderchar", "0x0E46" }, 874 { "extenderchar", "0x0EC6" }, 875 { "extenderchar", "0x3005" }, 876 { "extenderchar", "0x3031", Token.UPTO, "0x3035" }, 877 { "extenderchar", "0x309D", Token.UPTO, "0x309E" }, 878 { "extenderchar", "0x30FC", Token.UPTO, "0x30FE" }, 879 }; 880 881 882 883 public static final String [][] digitRules = { 884 { "octdigit", "'0'", Token.UPTO, "'7'" }, 885 { "octdigits", "octdigits", "octdigit" }, 886 { "octdigits", "octdigit" }, 887 888 { "bindigit", "'0'", Token.UPTO, "'1'" }, 889 { "bindigits", "bindigits", "bindigit" }, 890 { "bindigits", "bindigit" }, 891 }; 892 893 894 public static final String [][] numberRules = { 895 { "number", "float" }, 897 { "number", "integer" }, 898 { "float", "wholenumber", "'.'", "mantissa", "float_opt" }, 899 { "wholenumber", "digits" }, 900 { "wholenumber" }, 901 { "mantissa", "digits", "mantissa_opt" }, 902 { "mantissa_opt", "exponent", "digits" }, 903 { "mantissa_opt" }, 904 { "exponent", "exponentletter", "exponentsign" }, 905 { "exponentletter", "'e'" }, 906 { "exponentletter", "'E'" }, 907 { "exponentsign", "'-'" }, 908 { "exponentsign", "'+'" }, 909 { "exponentsign" }, 910 { "float_opt", "'f'" }, 911 { "float_opt", "'F'" }, 912 { "float_opt", "'d'" }, 913 { "float_opt", "'D'" }, 914 { "float_opt" }, 915 { "integer", "\"0X\"", "hexdigits" }, 916 { "integer", "\"0x\"", "hexdigits" }, 917 { "integer", "digits", "integer_opt" }, 918 { "integer_opt", "'l'" }, { "integer_opt", "'L'" }, { "integer_opt" }, 921 }; 922 923 924 public static final String [][] newlinesRules = { 925 { "newlines", "newlines", "newline" }, 926 { "newlines", "newline" }, 927 }; 928 929 930 public static final String [][] chardefRules = { 931 { "chardef", "\"'\\\"", "'0'", Token.UPTO, "'3'", "octdigit", "octdigit", "\"'\"" }, 932 { "chardef", "bnf_chardef" }, }; 934 935 936 public static final String [][] whitespaceRules = { 937 { "whitespace", "newline" }, 938 { "whitespace", "space" }, 939 { "whitespaces", "whitespaces", "whitespace" }, 940 { "whitespaces", "whitespace" }, 941 }; 942 943 944 private StandardLexerRules() {} 945 946 } 947 | Popular Tags |