1 23 24 package org.gjt.sp.jedit.syntax; 25 26 import javax.swing.text.Segment ; 28 import java.util.*; 29 import java.util.regex.Matcher ; 30 import java.util.regex.Pattern ; 31 import org.gjt.sp.util.SegmentCharSequence; 32 import org.gjt.sp.util.StandardUtilities; 33 35 46 public class TokenMarker 47 { 48 public TokenMarker() 50 {} 52 public void addRuleSet(ParserRuleSet rules) 54 { 55 ruleSets.put(rules.getSetName(), rules); 56 57 if (rules.getSetName().equals("MAIN")) 58 mainRuleSet = rules; 59 } 61 public ParserRuleSet getMainRuleSet() 63 { 64 return mainRuleSet; 65 } 67 public ParserRuleSet getRuleSet(String setName) 69 { 70 return ruleSets.get(setName); 71 } 73 77 public ParserRuleSet[] getRuleSets() 78 { 79 return ruleSets.values().toArray(new ParserRuleSet[ruleSets.size()]); 80 } 82 86 public LineContext markTokens(LineContext prevContext, 87 TokenHandler tokenHandler, Segment line) 88 { 89 this.tokenHandler = tokenHandler; 93 this.line = line; 94 95 lastOffset = line.offset; 96 lineLength = line.count + line.offset; 97 98 context = new LineContext(); 99 100 if(prevContext == null) 101 context.rules = getMainRuleSet(); 102 else 103 { 104 context.parent = prevContext.parent; 105 context.inRule = prevContext.inRule; 106 context.rules = prevContext.rules; 107 context.spanEndSubst = prevContext.spanEndSubst; 108 } 109 110 keywords = context.rules.getKeywords(); 111 escaped = false; 112 113 seenWhitespaceEnd = false; 114 whitespaceEnd = line.offset; 115 117 int terminateChar = context.rules.getTerminateChar(); 119 boolean terminated = false; 120 main_loop: for(pos = line.offset; pos < lineLength; pos++) 121 { 122 if(terminateChar >= 0 && pos - line.offset >= terminateChar 124 && !terminated) 125 { 126 terminated = true; 127 context = new LineContext(ParserRuleSet 128 .getStandardRuleSet(context.rules 129 .getDefault()),context); 130 keywords = context.rules.getKeywords(); 131 } 133 if(context.parent != null) 135 { 136 ParserRule rule = context.parent.inRule; 137 if(rule != null) 138 { 139 if(checkDelegateEnd(rule)) 140 { 141 seenWhitespaceEnd = true; 142 continue main_loop; 143 } 144 } 145 } 147 Character ch = Character.valueOf(line.array[pos]); 149 List<ParserRule> rules = context.rules.getRules(ch); 150 for (ParserRule rule : rules) 151 { 152 if (handleRule(rule,false,false)) 154 { 155 escape_checking: if ((rule.action & ParserRule.IS_ESCAPE) == ParserRule.IS_ESCAPE) 156 { 157 int escapeSequenceCount = pattern.count; 158 for (ParserRule innerRule : rules) 159 { 160 if (((innerRule.action & ParserRule.IS_ESCAPE) != ParserRule.IS_ESCAPE) && 161 (handleRule(innerRule,false))) 162 { 163 break escape_checking; 164 } 165 } escaped = !escaped; 167 pos += escapeSequenceCount - 1; 168 } 169 seenWhitespaceEnd = true; 170 continue main_loop; 171 } 172 } 174 if(Character.isWhitespace(ch)) 176 { 177 if(!seenWhitespaceEnd) 178 whitespaceEnd = pos + 1; 179 180 if(context.inRule != null) 181 handleRule(context.inRule,true); 182 183 handleNoWordBreak(); 184 185 markKeyword(false); 186 187 if(lastOffset != pos) 188 { 189 tokenHandler.handleToken(line, 190 context.rules.getDefault(), 191 lastOffset - line.offset, 192 pos - lastOffset, 193 context); 194 } 195 196 tokenHandler.handleToken(line, 197 context.rules.getDefault(), 198 pos - line.offset,1,context); 199 lastOffset = pos + 1; 200 201 escaped = false; 202 } 203 else 204 { 205 if(keywords != null || context.rules.getRuleCount() != 0) 206 { 207 String noWordSep = context.rules.getNoWordSep(); 208 209 if(!Character.isLetterOrDigit(ch) 210 && noWordSep.indexOf(ch) == -1) 211 { 212 if(context.inRule != null) 213 handleRule(context.inRule,true); 214 215 handleNoWordBreak(); 216 217 markKeyword(true); 218 219 tokenHandler.handleToken(line, 220 context.rules.getDefault(), 221 lastOffset - line.offset,1, 222 context); 223 lastOffset = pos + 1; 224 } 225 } 226 227 seenWhitespaceEnd = true; 228 escaped = false; 229 } } 232 pos = lineLength; 234 235 if(context.inRule != null) 236 handleRule(context.inRule,true); 237 238 handleNoWordBreak(); 239 markKeyword(true); 240 242 unwind: while(context.parent != null) 244 { 245 ParserRule rule = context.parent.inRule; 246 if((rule != null && (rule.action 247 & ParserRule.NO_LINE_BREAK) == ParserRule.NO_LINE_BREAK) 248 || terminated) 249 { 250 context = context.parent; 251 keywords = context.rules.getKeywords(); 252 context.inRule = null; 253 } 254 else 255 break unwind; 256 } 258 tokenHandler.handleToken(line,Token.END, 259 pos - line.offset,0,context); 260 261 context = context.intern(); 262 tokenHandler.setLineContext(context); 263 264 265 this.line = null; 266 267 return context; 268 } 270 272 private final Map<String , ParserRuleSet> ruleSets = new Hashtable<String , ParserRuleSet>(64); 274 private ParserRuleSet mainRuleSet; 275 276 private TokenHandler tokenHandler; 279 280 private Segment line; 281 private LineContext context; 282 private KeywordMap keywords; 283 private final Segment pattern = new Segment (); 284 private int lastOffset; 285 private int lineLength; 286 private int pos; 287 private boolean escaped; 288 289 private int whitespaceEnd; 290 private boolean seenWhitespaceEnd; 291 293 private boolean checkDelegateEnd(ParserRule rule) 295 { 296 if(rule.end == null) 297 return false; 298 299 LineContext tempContext = context; 300 context = context.parent; 301 keywords = context.rules.getKeywords(); 302 boolean tempEscaped = escaped; 303 boolean b = handleRule(rule,true); 304 context = tempContext; 305 keywords = context.rules.getKeywords(); 306 307 if(b && !tempEscaped) 308 { 309 if(context.inRule != null) 310 handleRule(context.inRule,true); 311 312 markKeyword(true); 313 314 context = (LineContext)context.parent.clone(); 315 316 tokenHandler.handleToken(line, 317 (context.inRule.action & ParserRule.EXCLUDE_MATCH) 318 == ParserRule.EXCLUDE_MATCH 319 ? context.rules.getDefault() 320 : context.inRule.token, 321 pos - line.offset,pattern.count,context); 322 323 keywords = context.rules.getKeywords(); 324 context.inRule = null; 325 lastOffset = pos + pattern.count; 326 327 pos += pattern.count - 1; 329 330 return true; 331 } 332 333 if((rule.action & ParserRule.NO_ESCAPE) == 0) 335 { 336 ParserRule escape = context.parent.rules.getEscapeRule(); 337 escape_checking: if (escape != null && handleRule(escape,false,false)) 338 { 339 int escapeSequenceCount = pattern.count; 340 Character ch = Character.valueOf(escape.upHashChar.charAt(0)); 341 List<ParserRule> rules = context.rules.getRules(ch); 342 for (ParserRule innerRule : rules) 343 { 344 if (handleRule(innerRule,false)) 345 { 346 break escape_checking; 347 } 348 } escaped = !escaped; 350 pos += escapeSequenceCount - 1; 351 return true; 352 } 353 } 354 355 return false; 356 } 358 363 private boolean handleRule(ParserRule checkRule, boolean end) 364 { 365 return handleRule(checkRule,end,true); 366 } 368 373 private boolean handleRule(ParserRule checkRule, boolean end, boolean processEscape) 374 { 375 if(!end) 377 { 378 if (null == checkRule.upHashChars) 379 { 380 if ((null != checkRule.upHashChar) && 381 (pos + checkRule.upHashChar.length() < line.array.length) && 382 !checkRule.upHashChar.equals(new String (line.array,pos,checkRule.upHashChar.length()).toUpperCase())) 383 { 384 return false; 385 } 386 } 387 else 388 { 389 if (-1 == new String (checkRule.upHashChars).indexOf(Character.toUpperCase(line.array[pos]))) 390 { 391 return false; 392 } 393 } 394 } 395 396 int offset = ((checkRule.action & ParserRule.MARK_PREVIOUS) != 0) ? 397 lastOffset : pos; 398 int posMatch = end ? checkRule.endPosMatch : checkRule.startPosMatch; 399 400 if((posMatch & ParserRule.AT_LINE_START) 401 == ParserRule.AT_LINE_START) 402 { 403 if(offset != line.offset) 404 { 405 return false; 406 } 407 } 408 else if((posMatch & ParserRule.AT_WHITESPACE_END) 409 == ParserRule.AT_WHITESPACE_END) 410 { 411 if(offset != whitespaceEnd) 412 { 413 return false; 414 } 415 } 416 else if((posMatch & ParserRule.AT_WORD_START) 417 == ParserRule.AT_WORD_START) 418 { 419 if(offset != lastOffset) 420 { 421 return false; 422 } 423 } 425 int matchedChars = 1; 426 CharSequence charSeq = null; 427 Matcher match = null; 428 429 if(!end || (checkRule.action & ParserRule.MARK_FOLLOWING) == 0) 431 { 432 if((checkRule.action & ParserRule.REGEXP) == 0 || end) 434 { 435 if(end) 436 { 437 if(context.spanEndSubst != null) 438 pattern.array = context.spanEndSubst; 439 else 440 pattern.array = checkRule.end; 441 } 442 else 443 pattern.array = checkRule.start; 444 pattern.offset = 0; 445 pattern.count = pattern.array.length; 446 matchedChars = pattern.count; 447 448 if(!SyntaxUtilities.regionMatches(context.rules 449 .getIgnoreCase(),line,pos,pattern.array)) 450 { 451 return false; 452 } 453 } 454 else 455 { 456 charSeq = new SegmentCharSequence(line, pos - line.offset, 460 line.count - (pos - line.offset)); 461 match = checkRule.startRegexp.matcher(charSeq); 462 if(!match.lookingAt()) 463 { 464 return false; 465 } 466 else if(match.start() != 0) 467 { 468 throw new InternalError ("Can't happen"); 469 } 470 else 471 { 472 matchedChars = match.end(); 473 476 if(matchedChars == 0) 477 matchedChars = 1; 478 } 479 } 480 } if((checkRule.action & ParserRule.IS_ESCAPE) == ParserRule.IS_ESCAPE) 483 { 484 if(context.inRule != null) 485 handleRule(context.inRule,true); 486 if (processEscape) 487 { 488 escaped = !escaped; 489 pos += pattern.count - 1; 490 } 491 } 492 else if(escaped) 493 { 494 escaped = false; 495 pos += pattern.count - 1; 496 } else if(!end) 499 { 500 if(context.inRule != null) 501 handleRule(context.inRule,true); 502 503 markKeyword((checkRule.action & ParserRule.MARK_PREVIOUS) 504 != ParserRule.MARK_PREVIOUS); 505 506 switch(checkRule.action & ParserRule.MAJOR_ACTIONS) 507 { 508 case ParserRule.SEQ: 510 context.spanEndSubst = null; 511 512 if((checkRule.action & ParserRule.REGEXP) != 0) 513 { 514 handleTokenWithSpaces(tokenHandler, 515 checkRule.token, 516 pos - line.offset, 517 matchedChars, 518 context); 519 } 520 else 521 { 522 tokenHandler.handleToken(line, 523 checkRule.token, 524 pos - line.offset, 525 matchedChars,context); 526 } 527 528 if(checkRule.delegate != null) 531 { 532 context = new LineContext( 533 checkRule.delegate, 534 context.parent); 535 keywords = context.rules.getKeywords(); 536 } 537 break; 538 case ParserRule.SPAN: 541 case ParserRule.EOL_SPAN: 542 context.inRule = checkRule; 543 544 byte tokenType = (checkRule.action & ParserRule.EXCLUDE_MATCH) 545 == ParserRule.EXCLUDE_MATCH 546 ? context.rules.getDefault() : checkRule.token; 547 548 if((checkRule.action & ParserRule.REGEXP) != 0) 549 { 550 handleTokenWithSpaces(tokenHandler, 551 tokenType, 552 pos - line.offset, 553 matchedChars, 554 context); 555 } 556 else 557 { 558 tokenHandler.handleToken(line,tokenType, 559 pos - line.offset, 560 matchedChars,context); 561 } 562 563 char[] spanEndSubst = null; 564 573 if(charSeq != null && checkRule.end != null) 574 { 575 spanEndSubst = substitute(match, 576 checkRule.end); 577 } 578 579 context.spanEndSubst = spanEndSubst; 580 context = new LineContext( 581 checkRule.delegate, 582 context); 583 keywords = context.rules.getKeywords(); 584 585 break; 586 case ParserRule.MARK_FOLLOWING: 589 tokenHandler.handleToken(line,(checkRule.action 590 & ParserRule.EXCLUDE_MATCH) 591 == ParserRule.EXCLUDE_MATCH ? 592 context.rules.getDefault() 593 : checkRule.token,pos - line.offset, 594 pattern.count,context); 595 596 context.spanEndSubst = null; 597 context.inRule = checkRule; 598 break; 599 case ParserRule.MARK_PREVIOUS: 602 context.spanEndSubst = null; 603 604 if ((checkRule.action & ParserRule.EXCLUDE_MATCH) 605 == ParserRule.EXCLUDE_MATCH) 606 { 607 if(pos != lastOffset) 608 { 609 tokenHandler.handleToken(line, 610 checkRule.token, 611 lastOffset - line.offset, 612 pos - lastOffset, 613 context); 614 } 615 616 tokenHandler.handleToken(line, 617 context.rules.getDefault(), 618 pos - line.offset,pattern.count, 619 context); 620 } 621 else 622 { 623 tokenHandler.handleToken(line, 624 checkRule.token, 625 lastOffset - line.offset, 626 pos - lastOffset + pattern.count, 627 context); 628 } 629 630 break; 631 default: 633 throw new InternalError ("Unhandled major action"); 634 } 635 636 pos += matchedChars - 1; 638 lastOffset = pos + 1; 639 640 } else if((context.inRule.action & ParserRule.MARK_FOLLOWING) != 0) 644 { 645 if(pos != lastOffset) 646 { 647 tokenHandler.handleToken(line, 648 context.inRule.token, 649 lastOffset - line.offset, 650 pos - lastOffset,context); 651 } 652 653 lastOffset = pos; 654 context.inRule = null; 655 } 657 return true; 658 } 660 private void handleNoWordBreak() 662 { 663 if(context.parent != null) 664 { 665 ParserRule rule = context.parent.inRule; 666 if(rule != null && (context.parent.inRule.action 667 & ParserRule.NO_WORD_BREAK) != 0) 668 { 669 if(pos != lastOffset) 670 { 671 tokenHandler.handleToken(line, 672 rule.token, 673 lastOffset - line.offset, 674 pos - lastOffset,context); 675 } 676 677 lastOffset = pos; 678 context = context.parent; 679 keywords = context.rules.getKeywords(); 680 context.inRule = null; 681 } 682 } 683 } 685 private void handleTokenWithSpaces(TokenHandler tokenHandler, 687 byte tokenType, int start, int len, LineContext context) 688 { 689 int last = start; 690 int end = start + len; 691 692 for(int i = start; i < end; i++) 693 { 694 if(Character.isWhitespace(line.array[i + line.offset])) 695 { 696 if(last != i) 697 { 698 tokenHandler.handleToken(line, 699 tokenType,last,i - last,context); 700 } 701 tokenHandler.handleToken(line,tokenType,i,1,context); 702 last = i + 1; 703 } 704 } 705 706 if(last != end) 707 { 708 tokenHandler.handleToken(line,tokenType,last, 709 end - last,context); 710 } 711 } 713 private void markKeyword(boolean addRemaining) 715 { 716 int len = pos - lastOffset; 717 if(len == 0) 718 return; 719 720 if(context.rules.getHighlightDigits()) 722 { 723 boolean digit = false; 724 boolean mixed = false; 725 726 for(int i = lastOffset; i < pos; i++) 727 { 728 char ch = line.array[i]; 729 if(Character.isDigit(ch)) 730 digit = true; 731 else 732 mixed = true; 733 } 734 735 if(mixed) 736 { 737 Pattern digitRE = context.rules.getDigitRegexp(); 738 739 if(digit) 742 { 743 if(digitRE == null) 744 { 745 digit = false; 749 } 750 else 751 { 752 int oldCount = line.count; 753 int oldOffset = line.offset; 754 line.offset = lastOffset; 755 line.count = len; 756 CharSequence seq = new SegmentCharSequence(line); 757 digit = digitRE.matcher(seq).matches(); 758 line.offset = oldOffset; 759 line.count = oldCount; 760 } 761 } 762 } 763 764 if(digit) 765 { 766 tokenHandler.handleToken(line,Token.DIGIT, 767 lastOffset - line.offset, 768 len,context); 769 lastOffset = pos; 770 771 return; 772 } 773 } 775 if(keywords != null) 777 { 778 byte id = keywords.lookup(line, lastOffset, len); 779 780 if(id != Token.NULL) 781 { 782 tokenHandler.handleToken(line,id, 783 lastOffset - line.offset, 784 len,context); 785 lastOffset = pos; 786 return; 787 } 788 } 790 if(addRemaining) 792 { 793 tokenHandler.handleToken(line,context.rules.getDefault(), 794 lastOffset - line.offset,len,context); 795 lastOffset = pos; 796 } } 799 private static char[] substitute(Matcher match, char[] end) 801 { 802 StringBuilder buf = new StringBuilder (); 803 for(int i = 0; i < end.length; i++) 804 { 805 char ch = end[i]; 806 if(ch == '$') 807 { 808 if(i == end.length - 1) 809 buf.append(ch); 810 else 811 { 812 char digit = end[i + 1]; 813 if(!Character.isDigit(digit)) 814 buf.append(ch); 815 else 816 { 817 buf.append(match.group( 818 digit - '0')); 819 i++; 820 } 821 } 822 } 823 else 824 buf.append(ch); 825 } 826 827 char[] returnValue = new char[buf.length()]; 828 buf.getChars(0,buf.length(),returnValue,0); 829 return returnValue; 830 } 832 834 838 public static class LineContext 839 { 840 private static final Map<LineContext, LineContext> intern = new HashMap<LineContext, LineContext>(); 841 842 public LineContext parent; 843 public ParserRule inRule; 844 public ParserRuleSet rules; 845 public char[] spanEndSubst; 847 848 public LineContext(ParserRuleSet rs, LineContext lc) 850 { 851 rules = rs; 852 parent = (lc == null ? null : (LineContext)lc.clone()); 853 } 855 public LineContext() 857 { 858 } 860 public LineContext intern() 862 { 863 LineContext obj = intern.get(this); 864 if(obj == null) 865 { 866 intern.put(this,this); 867 return this; 868 } 869 else 870 return obj; 871 } 873 public int hashCode() 875 { 876 if(inRule != null) 877 return inRule.hashCode(); 878 else if(rules != null) 879 return rules.hashCode(); 880 else 881 return 0; 882 } 884 public boolean equals(Object obj) 886 { 887 if(obj instanceof LineContext) 888 { 889 LineContext lc = (LineContext)obj; 890 return lc.inRule == inRule && lc.rules == rules 891 && StandardUtilities.objectsEqual(parent,lc.parent) 892 && charArraysEqual(spanEndSubst,lc.spanEndSubst); 893 } 894 else 895 return false; 896 } 898 public Object clone() 900 { 901 LineContext lc = new LineContext(); 902 lc.inRule = inRule; 903 lc.rules = rules; 904 lc.parent = (parent == null) ? null : (LineContext) parent.clone(); 905 lc.spanEndSubst = spanEndSubst; 906 907 return lc; 908 } 910 private static boolean charArraysEqual(char[] c1, char[] c2) 912 { 913 if(c1 == null) 914 return c2 == null; 915 916 if(c2 == null) 918 return false; 919 920 if(c1.length != c2.length) 921 return false; 922 923 for(int i = 0; i < c1.length; i++) 924 { 925 if(c1[i] != c2[i]) 926 return false; 927 } 928 929 return true; 930 } } } 933 | Popular Tags |