1 36 38 package com.google.gwt.dev.js.rhino; 39 40 import java.io.*; 41 42 53 54 public class TokenStream { 55 56 static final boolean RESERVED_KEYWORD_AS_IDENTIFIER = false; 57 58 62 63 final static int 64 TSF_NEWLINES = 1 << 0, TSF_FUNCTION = 1 << 1, TSF_RETURN_EXPR = 1 << 2, TSF_RETURN_VOID = 1 << 3, TSF_REGEXP = 1 << 4, TSF_DIRTYLINE = 1 << 5; 72 77 private final static int 78 EOF_CHAR = -1; 79 80 84 85 public final static int 86 ERROR = -1, EOF = 0, EOL = 1, POPV = 2, 93 ENTERWITH = 3, 94 LEAVEWITH = 4, 95 RETURN = 5, 96 GOTO = 6, 97 IFEQ = 7, 98 IFNE = 8, 99 DUP = 9, 100 SETNAME = 10, 101 BITOR = 11, 102 BITXOR = 12, 103 BITAND = 13, 104 EQ = 14, 105 NE = 15, 106 LT = 16, 107 LE = 17, 108 GT = 18, 109 GE = 19, 110 LSH = 20, 111 RSH = 21, 112 URSH = 22, 113 ADD = 23, 114 SUB = 24, 115 MUL = 25, 116 DIV = 26, 117 MOD = 27, 118 BITNOT = 28, 119 NEG = 29, 120 NEW = 30, 121 DELPROP = 31, 122 TYPEOF = 32, 123 NAMEINC = 33, 124 PROPINC = 34, 125 ELEMINC = 35, 126 NAMEDEC = 36, 127 PROPDEC = 37, 128 ELEMDEC = 38, 129 GETPROP = 39, 130 SETPROP = 40, 131 GETELEM = 41, 132 SETELEM = 42, 133 CALL = 43, 134 NAME = 44, 135 NUMBER = 45, 136 STRING = 46, 137 ZERO = 47, 138 ONE = 48, 139 NULL = 49, 140 THIS = 50, 141 FALSE = 51, 142 TRUE = 52, 143 SHEQ = 53, SHNE = 54, CLOSURE = 55, 146 REGEXP = 56, 147 POP = 57, 148 POS = 58, 149 VARINC = 59, 150 VARDEC = 60, 151 BINDNAME = 61, 152 THROW = 62, 153 IN = 63, 154 INSTANCEOF = 64, 155 GOSUB = 65, 156 RETSUB = 66, 157 CALLSPECIAL = 67, 158 GETTHIS = 68, 159 NEWTEMP = 69, 160 USETEMP = 70, 161 GETBASE = 71, 162 GETVAR = 72, 163 SETVAR = 73, 164 UNDEFINED = 74, 165 TRY = 75, 166 ENDTRY = 76, 167 NEWSCOPE = 77, 168 TYPEOFNAME = 78, 169 ENUMINIT = 79, 170 ENUMNEXT = 80, 171 GETPROTO = 81, 172 GETPARENT = 82, 173 SETPROTO = 83, 174 SETPARENT = 84, 175 SCOPE = 85, 176 GETSCOPEPARENT = 86, 177 THISFN = 87, 178 JTHROW = 88, 179 SEMI = 89, LB = 90, RB = 91, 183 LC = 92, RC = 93, 185 LP = 94, GWT = 95, 187 COMMA = 96, ASSIGN = 97, HOOK = 98, COLON = 99, 191 OR = 100, AND = 101, EQOP = 102, RELOP = 103, SHOP = 104, UNARYOP = 105, INC = 106, DEC = 107, 199 DOT = 108, PRIMARY = 109, FUNCTION = 110, EXPORT = 111, IMPORT = 112, IF = 113, ELSE = 114, SWITCH = 115, CASE = 116, DEFAULT = 117, WHILE = 118, DO = 119, FOR = 120, BREAK = 121, CONTINUE = 122, VAR = 123, WITH = 124, CATCH = 125, FINALLY = 126, RESERVED = 127, 220 227 NOP = 128, NOT = 129, PRE = 130, POST = 131, 231 232 236 237 VOID = 132, 238 239 242 BLOCK = 133, ARRAYLIT = 134, OBJLIT = 135, LABEL = 136, TARGET = 137, 247 LOOP = 138, 248 ENUMDONE = 139, 249 EXPRSTMT = 140, 250 PARENT = 141, 251 CONVERT = 142, 252 JSR = 143, 253 NEWLOCAL = 144, 254 USELOCAL = 145, 255 DEBUGGER = 146, 256 SCRIPT = 147, 258 LAST_TOKEN = 147; 259 260 262 263 public static String tokenToName(int token) { 264 if (Context.printTrees || Context.printICode) { 265 switch (token) { 266 case ERROR: return "error"; 267 case EOF: return "eof"; 268 case EOL: return "eol"; 269 case POPV: return "popv"; 270 case ENTERWITH: return "enterwith"; 271 case LEAVEWITH: return "leavewith"; 272 case RETURN: return "return"; 273 case GOTO: return "goto"; 274 case IFEQ: return "ifeq"; 275 case IFNE: return "ifne"; 276 case DUP: return "dup"; 277 case SETNAME: return "setname"; 278 case BITOR: return "bitor"; 279 case BITXOR: return "bitxor"; 280 case BITAND: return "bitand"; 281 case EQ: return "eq"; 282 case NE: return "ne"; 283 case LT: return "lt"; 284 case LE: return "le"; 285 case GT: return "gt"; 286 case GE: return "ge"; 287 case LSH: return "lsh"; 288 case RSH: return "rsh"; 289 case URSH: return "ursh"; 290 case ADD: return "add"; 291 case SUB: return "sub"; 292 case MUL: return "mul"; 293 case DIV: return "div"; 294 case MOD: return "mod"; 295 case BITNOT: return "bitnot"; 296 case NEG: return "neg"; 297 case NEW: return "new"; 298 case DELPROP: return "delprop"; 299 case TYPEOF: return "typeof"; 300 case NAMEINC: return "nameinc"; 301 case PROPINC: return "propinc"; 302 case ELEMINC: return "eleminc"; 303 case NAMEDEC: return "namedec"; 304 case PROPDEC: return "propdec"; 305 case ELEMDEC: return "elemdec"; 306 case GETPROP: return "getprop"; 307 case SETPROP: return "setprop"; 308 case GETELEM: return "getelem"; 309 case SETELEM: return "setelem"; 310 case CALL: return "call"; 311 case NAME: return "name"; 312 case NUMBER: return "number"; 313 case STRING: return "string"; 314 case ZERO: return "zero"; 315 case ONE: return "one"; 316 case NULL: return "null"; 317 case THIS: return "this"; 318 case FALSE: return "false"; 319 case TRUE: return "true"; 320 case SHEQ: return "sheq"; 321 case SHNE: return "shne"; 322 case CLOSURE: return "closure"; 323 case REGEXP: return "object"; 324 case POP: return "pop"; 325 case POS: return "pos"; 326 case VARINC: return "varinc"; 327 case VARDEC: return "vardec"; 328 case BINDNAME: return "bindname"; 329 case THROW: return "throw"; 330 case IN: return "in"; 331 case INSTANCEOF: return "instanceof"; 332 case GOSUB: return "gosub"; 333 case RETSUB: return "retsub"; 334 case CALLSPECIAL: return "callspecial"; 335 case GETTHIS: return "getthis"; 336 case NEWTEMP: return "newtemp"; 337 case USETEMP: return "usetemp"; 338 case GETBASE: return "getbase"; 339 case GETVAR: return "getvar"; 340 case SETVAR: return "setvar"; 341 case UNDEFINED: return "undefined"; 342 case TRY: return "try"; 343 case ENDTRY: return "endtry"; 344 case NEWSCOPE: return "newscope"; 345 case TYPEOFNAME: return "typeofname"; 346 case ENUMINIT: return "enuminit"; 347 case ENUMNEXT: return "enumnext"; 348 case GETPROTO: return "getproto"; 349 case GETPARENT: return "getparent"; 350 case SETPROTO: return "setproto"; 351 case SETPARENT: return "setparent"; 352 case SCOPE: return "scope"; 353 case GETSCOPEPARENT: return "getscopeparent"; 354 case THISFN: return "thisfn"; 355 case JTHROW: return "jthrow"; 356 case SEMI: return "semi"; 357 case LB: return "lb"; 358 case RB: return "rb"; 359 case LC: return "lc"; 360 case RC: return "rc"; 361 case LP: return "lp"; 362 case GWT: return "gwt"; 363 case COMMA: return "comma"; 364 case ASSIGN: return "assign"; 365 case HOOK: return "hook"; 366 case COLON: return "colon"; 367 case OR: return "or"; 368 case AND: return "and"; 369 case EQOP: return "eqop"; 370 case RELOP: return "relop"; 371 case SHOP: return "shop"; 372 case UNARYOP: return "unaryop"; 373 case INC: return "inc"; 374 case DEC: return "dec"; 375 case DOT: return "dot"; 376 case PRIMARY: return "primary"; 377 case FUNCTION: return "function"; 378 case EXPORT: return "export"; 379 case IMPORT: return "import"; 380 case IF: return "if"; 381 case ELSE: return "else"; 382 case SWITCH: return "switch"; 383 case CASE: return "case"; 384 case DEFAULT: return "default"; 385 case WHILE: return "while"; 386 case DO: return "do"; 387 case FOR: return "for"; 388 case BREAK: return "break"; 389 case CONTINUE: return "continue"; 390 case VAR: return "var"; 391 case WITH: return "with"; 392 case CATCH: return "catch"; 393 case FINALLY: return "finally"; 394 case RESERVED: return "reserved"; 395 case NOP: return "nop"; 396 case NOT: return "not"; 397 case PRE: return "pre"; 398 case POST: return "post"; 399 case VOID: return "void"; 400 case BLOCK: return "block"; 401 case ARRAYLIT: return "arraylit"; 402 case OBJLIT: return "objlit"; 403 case LABEL: return "label"; 404 case TARGET: return "target"; 405 case LOOP: return "loop"; 406 case ENUMDONE: return "enumdone"; 407 case EXPRSTMT: return "exprstmt"; 408 case PARENT: return "parent"; 409 case CONVERT: return "convert"; 410 case JSR: return "jsr"; 411 case NEWLOCAL: return "newlocal"; 412 case USELOCAL: return "uselocal"; 413 case SCRIPT: return "script"; 414 } 415 return "<unknown="+token+">"; 416 } 417 return ""; 418 } 419 420 424 public String tokenToString(int token) { 425 if (Context.printTrees) { 426 String name = tokenToName(token); 427 428 switch (token) { 429 case UNARYOP: 430 case ASSIGN: 431 case PRIMARY: 432 case EQOP: 433 case SHOP: 434 case RELOP: 435 return name + " " + tokenToName(this.op); 436 437 case STRING: 438 case REGEXP: 439 case NAME: 440 return name + " `" + this.string + "'"; 441 442 case NUMBER: 443 return "NUMBER " + this.number; 444 } 445 446 return name; 447 } 448 return ""; 449 } 450 451 private static int getKeywordId(String name) { 452 final int 455 Id_break = BREAK, 456 Id_case = CASE, 457 Id_continue = CONTINUE, 458 Id_default = DEFAULT, 459 Id_delete = DELPROP, 460 Id_do = DO, 461 Id_else = ELSE, 462 Id_export = EXPORT, 463 Id_false = PRIMARY | (FALSE << 8), 464 Id_for = FOR, 465 Id_function = FUNCTION, 466 Id_if = IF, 467 Id_in = RELOP | (IN << 8), 468 Id_new = NEW, 469 Id_null = PRIMARY | (NULL << 8), 470 Id_return = RETURN, 471 Id_switch = SWITCH, 472 Id_this = PRIMARY | (THIS << 8), 473 Id_true = PRIMARY | (TRUE << 8), 474 Id_typeof = UNARYOP | (TYPEOF << 8), 475 Id_var = VAR, 476 Id_void = UNARYOP | (VOID << 8), 477 Id_while = WHILE, 478 Id_with = WITH, 479 480 Id_abstract = RESERVED, 482 Id_boolean = RESERVED, 483 Id_byte = RESERVED, 484 Id_catch = CATCH, 485 Id_char = RESERVED, 486 Id_class = RESERVED, 487 Id_const = RESERVED, 488 Id_debugger = DEBUGGER, 489 Id_double = RESERVED, 490 Id_enum = RESERVED, 491 Id_extends = RESERVED, 492 Id_final = RESERVED, 493 Id_finally = FINALLY, 494 Id_float = RESERVED, 495 Id_goto = RESERVED, 496 Id_implements = RESERVED, 497 Id_import = IMPORT, 498 Id_instanceof = RELOP | (INSTANCEOF << 8), 499 Id_int = RESERVED, 500 Id_interface = RESERVED, 501 Id_long = RESERVED, 502 Id_native = RESERVED, 503 Id_package = RESERVED, 504 Id_private = RESERVED, 505 Id_protected = RESERVED, 506 Id_public = RESERVED, 507 Id_short = RESERVED, 508 Id_static = RESERVED, 509 Id_super = RESERVED, 510 Id_synchronized = RESERVED, 511 Id_throw = THROW, 512 Id_throws = RESERVED, 513 Id_transient = RESERVED, 514 Id_try = TRY, 515 Id_volatile = RESERVED; 516 517 int id; 518 String s = name; 519 L0: { id = 0; String X = null; int c; 521 L: switch (s.length()) { 522 case 2: c=s.charAt(1); 523 if (c=='f') { if (s.charAt(0)=='i') {id=Id_if; break L0;} } 524 else if (c=='n') { if (s.charAt(0)=='i') {id=Id_in; break L0;} } 525 else if (c=='o') { if (s.charAt(0)=='d') {id=Id_do; break L0;} } 526 break L; 527 case 3: switch (s.charAt(0)) { 528 case 'f': if (s.charAt(2)=='r' && s.charAt(1)=='o') {id=Id_for; break L0;} break L; 529 case 'i': if (s.charAt(2)=='t' && s.charAt(1)=='n') {id=Id_int; break L0;} break L; 530 case 'n': if (s.charAt(2)=='w' && s.charAt(1)=='e') {id=Id_new; break L0;} break L; 531 case 't': if (s.charAt(2)=='y' && s.charAt(1)=='r') {id=Id_try; break L0;} break L; 532 case 'v': if (s.charAt(2)=='r' && s.charAt(1)=='a') {id=Id_var; break L0;} break L; 533 } break L; 534 case 4: switch (s.charAt(0)) { 535 case 'b': X="byte";id=Id_byte; break L; 536 case 'c': c=s.charAt(3); 537 if (c=='e') { if (s.charAt(2)=='s' && s.charAt(1)=='a') {id=Id_case; break L0;} } 538 else if (c=='r') { if (s.charAt(2)=='a' && s.charAt(1)=='h') {id=Id_char; break L0;} } 539 break L; 540 case 'e': c=s.charAt(3); 541 if (c=='e') { if (s.charAt(2)=='s' && s.charAt(1)=='l') {id=Id_else; break L0;} } 542 else if (c=='m') { if (s.charAt(2)=='u' && s.charAt(1)=='n') {id=Id_enum; break L0;} } 543 break L; 544 case 'g': X="goto";id=Id_goto; break L; 545 case 'l': X="long";id=Id_long; break L; 546 case 'n': X="null";id=Id_null; break L; 547 case 't': c=s.charAt(3); 548 if (c=='e') { if (s.charAt(2)=='u' && s.charAt(1)=='r') {id=Id_true; break L0;} } 549 else if (c=='s') { if (s.charAt(2)=='i' && s.charAt(1)=='h') {id=Id_this; break L0;} } 550 break L; 551 case 'v': X="void";id=Id_void; break L; 552 case 'w': X="with";id=Id_with; break L; 553 } break L; 554 case 5: switch (s.charAt(2)) { 555 case 'a': X="class";id=Id_class; break L; 556 case 'e': X="break";id=Id_break; break L; 557 case 'i': X="while";id=Id_while; break L; 558 case 'l': X="false";id=Id_false; break L; 559 case 'n': c=s.charAt(0); 560 if (c=='c') { X="const";id=Id_const; } 561 else if (c=='f') { X="final";id=Id_final; } 562 break L; 563 case 'o': c=s.charAt(0); 564 if (c=='f') { X="float";id=Id_float; } 565 else if (c=='s') { X="short";id=Id_short; } 566 break L; 567 case 'p': X="super";id=Id_super; break L; 568 case 'r': X="throw";id=Id_throw; break L; 569 case 't': X="catch";id=Id_catch; break L; 570 } break L; 571 case 6: switch (s.charAt(1)) { 572 case 'a': X="native";id=Id_native; break L; 573 case 'e': c=s.charAt(0); 574 if (c=='d') { X="delete";id=Id_delete; } 575 else if (c=='r') { X="return";id=Id_return; } 576 break L; 577 case 'h': X="throws";id=Id_throws; break L; 578 case 'm': X="import";id=Id_import; break L; 579 case 'o': X="double";id=Id_double; break L; 580 case 't': X="static";id=Id_static; break L; 581 case 'u': X="public";id=Id_public; break L; 582 case 'w': X="switch";id=Id_switch; break L; 583 case 'x': X="export";id=Id_export; break L; 584 case 'y': X="typeof";id=Id_typeof; break L; 585 } break L; 586 case 7: switch (s.charAt(1)) { 587 case 'a': X="package";id=Id_package; break L; 588 case 'e': X="default";id=Id_default; break L; 589 case 'i': X="finally";id=Id_finally; break L; 590 case 'o': X="boolean";id=Id_boolean; break L; 591 case 'r': X="private";id=Id_private; break L; 592 case 'x': X="extends";id=Id_extends; break L; 593 } break L; 594 case 8: switch (s.charAt(0)) { 595 case 'a': X="abstract";id=Id_abstract; break L; 596 case 'c': X="continue";id=Id_continue; break L; 597 case 'd': X="debugger";id=Id_debugger; break L; 598 case 'f': X="function";id=Id_function; break L; 599 case 'v': X="volatile";id=Id_volatile; break L; 600 } break L; 601 case 9: c=s.charAt(0); 602 if (c=='i') { X="interface";id=Id_interface; } 603 else if (c=='p') { X="protected";id=Id_protected; } 604 else if (c=='t') { X="transient";id=Id_transient; } 605 break L; 606 case 10: c=s.charAt(1); 607 if (c=='m') { X="implements";id=Id_implements; } 608 else if (c=='n') { X="instanceof";id=Id_instanceof; } 609 break L; 610 case 12: X="synchronized";id=Id_synchronized; break L; 611 } 612 if (X!=null && X!=s && !X.equals(s)) id = 0; 613 } 614 617 return id; 618 } 619 620 private int stringToKeyword(String name) { 621 int id = getKeywordId(name); 622 if (id == 0) { return EOF; } 623 this.op = id >> 8; 624 return id & 0xff; 625 } 626 627 public TokenStream(Reader in, 628 String sourceName, int lineno) 629 { 630 this.in = new LineBuffer(in, lineno); 631 this.pushbackToken = EOF; 632 this.sourceName = sourceName; 633 flags = 0; 634 } 635 636 639 public boolean matchToken(int toMatch) throws IOException { 640 int token = getToken(); 641 if (token == toMatch) 642 return true; 643 644 tokenno--; 646 this.pushbackToken = token; 647 return false; 648 } 649 650 public void clearPushback() { 651 this.pushbackToken = EOF; 652 } 653 654 public void ungetToken(int tt) { 655 if (this.pushbackToken != EOF && tt != ERROR) { 656 String message = Context.getMessage2("msg.token.replaces.pushback", 657 tokenToString(tt), tokenToString(this.pushbackToken)); 658 throw new RuntimeException (message); 659 } 660 this.pushbackToken = tt; 661 tokenno--; 662 } 663 664 public int peekToken() throws IOException { 665 int result = getToken(); 666 667 this.pushbackToken = result; 668 tokenno--; 669 return result; 670 } 671 672 public int peekTokenSameLine() throws IOException { 673 int result; 674 675 flags |= TSF_NEWLINES; result = peekToken(); 677 flags &= ~TSF_NEWLINES; if (this.pushbackToken == EOL) 679 this.pushbackToken = EOF; 680 return result; 681 } 682 683 public static boolean isJSKeyword(String s) { 684 return getKeywordId(s) != 0; 685 } 686 687 public static boolean isJSIdentifier(String s) { 688 int length = s.length(); 689 690 if (length == 0 || !Character.isJavaIdentifierStart(s.charAt(0))) 691 return false; 692 693 for (int i=1; i<length; i++) { 694 char c = s.charAt(i); 695 if (!Character.isJavaIdentifierPart(c)) { 696 if (c == '\\') { 697 if (! ((i + 5) < length) 698 && (s.charAt(i + 1) == 'u') 699 && 0 <= xDigitToInt(s.charAt(i + 2)) 700 && 0 <= xDigitToInt(s.charAt(i + 3)) 701 && 0 <= xDigitToInt(s.charAt(i + 4)) 702 && 0 <= xDigitToInt(s.charAt(i + 5))) { 703 return true; 704 } 705 } 706 707 return false; 708 } 709 } 710 711 return true; 712 } 713 714 private static boolean isAlpha(int c) { 715 return ((c >= 'a' && c <= 'z') 716 || (c >= 'A' && c <= 'Z')); 717 } 718 719 static boolean isDigit(int c) { 720 return (c >= '0' && c <= '9'); 721 } 722 723 static int xDigitToInt(int c) { 724 if ('0' <= c && c <= '9') { return c - '0'; } 725 if ('a' <= c && c <= 'f') { return c - ('a' - 10); } 726 if ('A' <= c && c <= 'F') { return c - ('A' - 10); } 727 return -1; 728 } 729 730 735 public static boolean isJSSpace(int c) { 736 return (c == '\u0020' || c == '\u0009' 737 || c == '\u000C' || c == '\u000B' 738 || c == '\u00A0' 739 || Character.getType((char)c) == Character.SPACE_SEPARATOR); 740 } 741 742 public static boolean isJSLineTerminator(int c) { 743 return (c == '\n' || c == '\r' 744 || c == 0x2028 || c == 0x2029); 745 } 746 747 private void skipLine() throws IOException { 748 int c; 750 while ((c = in.read()) != EOF_CHAR && c != '\n') { } 751 in.unread(); 752 } 753 754 public int getToken() throws IOException { 755 int c; 756 tokenno++; 757 758 if (this.pushbackToken != EOF) { 760 int result = this.pushbackToken; 761 this.pushbackToken = EOF; 762 return result; 763 } 764 765 do { 767 c = in.read(); 768 if (c == '\n') { 769 flags &= ~TSF_DIRTYLINE; 770 if ((flags & TSF_NEWLINES) != 0) 771 break; 772 } 773 } while (isJSSpace(c) || c == '\n'); 774 775 if (c == EOF_CHAR) 776 return EOF; 777 if (c != '-' && c != '\n') 778 flags |= TSF_DIRTYLINE; 779 780 boolean identifierStart; 783 boolean isUnicodeEscapeStart = false; 784 if (c == '\\') { 785 c = in.read(); 786 if (c == 'u') { 787 identifierStart = true; 788 isUnicodeEscapeStart = true; 789 stringBufferTop = 0; 790 } else { 791 identifierStart = false; 792 c = '\\'; 793 in.unread(); 794 } 795 } else { 796 identifierStart = Character.isJavaIdentifierStart((char)c); 797 if (identifierStart) { 798 stringBufferTop = 0; 799 addToString(c); 800 } 801 802 if (c == '@') { 806 stringBufferTop = 0; 807 addToString(c); 808 return jsniMatchReference(); 809 } 810 } 811 812 if (identifierStart) { 813 boolean containsEscape = isUnicodeEscapeStart; 814 for (;;) { 815 if (isUnicodeEscapeStart) { 816 int escapeVal = 0; 823 for (int i = 0; i != 4; ++i) { 824 c = in.read(); 825 escapeVal = (escapeVal << 4) | xDigitToInt(c); 826 if (escapeVal < 0) { break; } 828 } 829 if (escapeVal < 0) { 830 reportSyntaxError("msg.invalid.escape", null); 831 return ERROR; 832 } 833 addToString(escapeVal); 834 isUnicodeEscapeStart = false; 835 } else { 836 c = in.read(); 837 if (c == '\\') { 838 c = in.read(); 839 if (c == 'u') { 840 isUnicodeEscapeStart = true; 841 containsEscape = true; 842 } else { 843 reportSyntaxError("msg.illegal.character", null); 844 return ERROR; 845 } 846 } else { 847 if (!Character.isJavaIdentifierPart((char)c)) { 848 break; 849 } 850 addToString(c); 851 } 852 } 853 } 854 in.unread(); 855 856 String str = getStringFromBuffer(); 857 if (!containsEscape) { 858 861 int result = stringToKeyword(str); 863 if (result != EOF) { 864 if (result != RESERVED) { 865 return result; 866 } 867 else if (!RESERVED_KEYWORD_AS_IDENTIFIER) 868 { 869 return result; 870 } 871 else { 872 Object [] errArgs = { str }; 876 reportSyntaxWarning("msg.reserved.keyword", errArgs); 877 } 878 } 879 } 880 this.string = str; 881 return NAME; 882 } 883 884 if (isDigit(c) || (c == '.' && isDigit(in.peek()))) { 886 887 stringBufferTop = 0; 888 int base = 10; 889 890 if (c == '0') { 891 c = in.read(); 892 if (c == 'x' || c == 'X') { 893 base = 16; 894 c = in.read(); 895 } else if (isDigit(c)) { 896 base = 8; 897 } else { 898 addToString('0'); 899 } 900 } 901 902 if (base == 16) { 903 while (0 <= xDigitToInt(c)) { 904 addToString(c); 905 c = in.read(); 906 } 907 } else { 908 while ('0' <= c && c <= '9') { 909 915 if (base == 8 && c >= '8') { 916 Object [] errArgs = { c == '8' ? "8" : "9" }; 917 reportSyntaxWarning("msg.bad.octal.literal", errArgs); 918 base = 10; 919 } 920 addToString(c); 921 c = in.read(); 922 } 923 } 924 925 boolean isInteger = true; 926 927 if (base == 10 && (c == '.' || c == 'e' || c == 'E')) { 928 isInteger = false; 929 if (c == '.') { 930 do { 931 addToString(c); 932 c = in.read(); 933 } while (isDigit(c)); 934 } 935 if (c == 'e' || c == 'E') { 936 addToString(c); 937 c = in.read(); 938 if (c == '+' || c == '-') { 939 addToString(c); 940 c = in.read(); 941 } 942 if (!isDigit(c)) { 943 reportSyntaxError("msg.missing.exponent", null); 944 return ERROR; 945 } 946 do { 947 addToString(c); 948 c = in.read(); 949 } while (isDigit(c)); 950 } 951 } 952 in.unread(); 953 String numString = getStringFromBuffer(); 954 955 double dval; 956 if (base == 10 && !isInteger) { 957 try { 958 dval = (Double.valueOf(numString)).doubleValue(); 960 } 961 catch (NumberFormatException ex) { 962 Object [] errArgs = { ex.getMessage() }; 963 reportSyntaxError("msg.caught.nfe", errArgs); 964 return ERROR; 965 } 966 } else { 967 dval = ScriptRuntime.stringToNumber(numString, 0, base); 968 } 969 970 this.number = dval; 971 return NUMBER; 972 } 973 974 if (c == '"' || c == '\'') { 976 981 int quoteChar = c; 982 int val = 0; 983 stringBufferTop = 0; 984 985 c = in.read(); 986 strLoop: while (c != quoteChar) { 987 if (c == '\n' || c == EOF_CHAR) { 988 in.unread(); 989 reportSyntaxError("msg.unterminated.string.lit", null); 990 return ERROR; 991 } 992 993 if (c == '\\') { 994 996 c = in.read(); 997 switch (c) { 998 case 'b': c = '\b'; break; 999 case 'f': c = '\f'; break; 1000 case 'n': c = '\n'; break; 1001 case 'r': c = '\r'; break; 1002 case 't': c = '\t'; break; 1003 1004 case 'v': c = 0xb; break; 1007 1008 case 'u': { 1009 1014 int escapeStart = stringBufferTop; 1015 addToString('u'); 1016 int escapeVal = 0; 1017 for (int i = 0; i != 4; ++i) { 1018 c = in.read(); 1019 escapeVal = (escapeVal << 4) | xDigitToInt(c); 1020 if (escapeVal < 0) { 1021 continue strLoop; 1022 } 1023 addToString(c); 1024 } 1025 stringBufferTop = escapeStart; 1028 c = escapeVal; 1029 } break; 1030 1031 case 'x': { 1032 1035 c = in.read(); 1036 int escapeVal = xDigitToInt(c); 1037 if (escapeVal < 0) { 1038 addToString('x'); 1039 continue strLoop; 1040 } else { 1041 int c1 = c; 1042 c = in.read(); 1043 escapeVal = (escapeVal << 4) | xDigitToInt(c); 1044 if (escapeVal < 0) { 1045 addToString('x'); 1046 addToString(c1); 1047 continue strLoop; 1048 } else { 1049 c = escapeVal; 1051 } 1052 } 1053 } break; 1054 1055 case '\n': 1056 c = in.read(); 1058 continue strLoop; 1059 1060 default: if ('0' <= c && c < '8') { 1061 val = c - '0'; 1062 c = in.read(); 1063 if ('0' <= c && c < '8') { 1064 val = 8 * val + c - '0'; 1065 c = in.read(); 1066 if ('0' <= c && c < '8' && val <= 037) { 1067 val = 8 * val + c - '0'; 1070 c = in.read(); 1071 } 1072 } 1073 in.unread(); 1074 c = val; 1075 } 1076 } 1077 } 1078 addToString(c); 1079 c = in.read(); 1080 } 1081 1082 this.string = getStringFromBuffer(); 1083 return STRING; 1084 } 1085 1086 switch (c) 1087 { 1088 case '\n': return EOL; 1089 case ';': return SEMI; 1090 case '[': return LB; 1091 case ']': return RB; 1092 case '{': return LC; 1093 case '}': return RC; 1094 case '(': return LP; 1095 case ')': return GWT; 1096 case ',': return COMMA; 1097 case '?': return HOOK; 1098 case ':': return COLON; 1099 case '.': return DOT; 1100 1101 case '|': 1102 if (in.match('|')) { 1103 return OR; 1104 } else if (in.match('=')) { 1105 this.op = BITOR; 1106 return ASSIGN; 1107 } else { 1108 return BITOR; 1109 } 1110 1111 case '^': 1112 if (in.match('=')) { 1113 this.op = BITXOR; 1114 return ASSIGN; 1115 } else { 1116 return BITXOR; 1117 } 1118 1119 case '&': 1120 if (in.match('&')) { 1121 return AND; 1122 } else if (in.match('=')) { 1123 this.op = BITAND; 1124 return ASSIGN; 1125 } else { 1126 return BITAND; 1127 } 1128 1129 case '=': 1130 if (in.match('=')) { 1131 if (in.match('=')) 1132 this.op = SHEQ; 1133 else 1134 this.op = EQ; 1135 return EQOP; 1136 } else { 1137 this.op = NOP; 1138 return ASSIGN; 1139 } 1140 1141 case '!': 1142 if (in.match('=')) { 1143 if (in.match('=')) 1144 this.op = SHNE; 1145 else 1146 this.op = NE; 1147 return EQOP; 1148 } else { 1149 this.op = NOT; 1150 return UNARYOP; 1151 } 1152 1153 case '<': 1154 1155 if (in.match('!')) { 1156 if (in.match('-')) { 1157 if (in.match('-')) { 1158 skipLine(); 1159 return getToken(); } 1161 in.unread(); 1162 } 1163 in.unread(); 1164 } 1165 if (in.match('<')) { 1166 if (in.match('=')) { 1167 this.op = LSH; 1168 return ASSIGN; 1169 } else { 1170 this.op = LSH; 1171 return SHOP; 1172 } 1173 } else { 1174 if (in.match('=')) { 1175 this.op = LE; 1176 return RELOP; 1177 } else { 1178 this.op = LT; 1179 return RELOP; 1180 } 1181 } 1182 1183 case '>': 1184 if (in.match('>')) { 1185 if (in.match('>')) { 1186 if (in.match('=')) { 1187 this.op = URSH; 1188 return ASSIGN; 1189 } else { 1190 this.op = URSH; 1191 return SHOP; 1192 } 1193 } else { 1194 if (in.match('=')) { 1195 this.op = RSH; 1196 return ASSIGN; 1197 } else { 1198 this.op = RSH; 1199 return SHOP; 1200 } 1201 } 1202 } else { 1203 if (in.match('=')) { 1204 this.op = GE; 1205 return RELOP; 1206 } else { 1207 this.op = GT; 1208 return RELOP; 1209 } 1210 } 1211 1212 case '*': 1213 if (in.match('=')) { 1214 this.op = MUL; 1215 return ASSIGN; 1216 } else { 1217 return MUL; 1218 } 1219 1220 case '/': 1221 if (in.match('/')) { 1223 skipLine(); 1224 return getToken(); 1225 } 1226 if (in.match('*')) { 1227 while ((c = in.read()) != -1 && 1228 !(c == '*' && in.match('/'))) { 1229 ; } 1231 if (c == EOF_CHAR) { 1232 reportSyntaxError("msg.unterminated.comment", null); 1233 return ERROR; 1234 } 1235 return getToken(); } 1237 1238 if ((flags & TSF_REGEXP) != 0) { 1240 stringBufferTop = 0; 1241 while ((c = in.read()) != '/') { 1242 if (c == '\n' || c == EOF_CHAR) { 1243 in.unread(); 1244 reportSyntaxError("msg.unterminated.re.lit", null); 1245 return ERROR; 1246 } 1247 if (c == '\\') { 1248 addToString(c); 1249 c = in.read(); 1250 } 1251 1252 addToString(c); 1253 } 1254 int reEnd = stringBufferTop; 1255 1256 while (true) { 1257 if (in.match('g')) 1258 addToString('g'); 1259 else if (in.match('i')) 1260 addToString('i'); 1261 else if (in.match('m')) 1262 addToString('m'); 1263 else 1264 break; 1265 } 1266 1267 if (isAlpha(in.peek())) { 1268 reportSyntaxError("msg.invalid.re.flag", null); 1269 return ERROR; 1270 } 1271 1272 this.string = new String (stringBuffer, 0, reEnd); 1273 this.regExpFlags = new String (stringBuffer, reEnd, 1274 stringBufferTop - reEnd); 1275 return REGEXP; 1276 } 1277 1278 1279 if (in.match('=')) { 1280 this.op = DIV; 1281 return ASSIGN; 1282 } else { 1283 return DIV; 1284 } 1285 1286 case '%': 1287 this.op = MOD; 1288 if (in.match('=')) { 1289 return ASSIGN; 1290 } else { 1291 return MOD; 1292 } 1293 1294 case '~': 1295 this.op = BITNOT; 1296 return UNARYOP; 1297 1298 case '+': 1299 if (in.match('=')) { 1300 this.op = ADD; 1301 return ASSIGN; 1302 } else if (in.match('+')) { 1303 return INC; 1304 } else { 1305 return ADD; 1306 } 1307 1308 case '-': 1309 if (in.match('=')) { 1310 this.op = SUB; 1311 c = ASSIGN; 1312 } else if (in.match('-')) { 1313 if (0 == (flags & TSF_DIRTYLINE)) { 1314 if (in.match('>')) { 1317 skipLine(); 1318 return getToken(); 1319 } 1320 } 1321 c = DEC; 1322 } else { 1323 c = SUB; 1324 } 1325 flags |= TSF_DIRTYLINE; 1326 return c; 1327 1328 default: 1329 reportSyntaxError("msg.illegal.character", null); 1330 return ERROR; 1331 } 1332 } 1333 1334 private int jsniMatchReference() throws IOException { 1335 1336 if (!jsniMatchQualifiedTypeName('.', ':')) { 1338 return ERROR; 1339 } 1340 1341 int c = in.read(); 1344 if (c != ':') { 1345 in.unread(); 1346 reportSyntaxError("msg.jsni.expected.char", new String [] { ":" }); 1347 return ERROR; 1348 } 1349 addToString(c); 1350 1351 if (!jsniMatchMethodSignatureOrFieldName()) { 1354 return ERROR; 1355 } 1356 1357 this.string = new String (stringBuffer, 0, stringBufferTop); 1358 return NAME; 1359 } 1360 1361 private boolean jsniMatchParamListSignature() throws IOException { 1362 do { 1366 int c = in.read(); 1367 if (c == ')') { 1368 addToString(c); 1371 return true; 1372 } 1373 1374 in.unread(); 1375 } while (jsniMatchParamTypeSignature()); 1376 1377 return false; 1382 } 1383 1384 private boolean jsniMatchParamTypeSignature() throws IOException { 1385 int c = in.read(); 1386 switch (c) { 1387 case 'Z': 1388 case 'B': 1389 case 'C': 1390 case 'S': 1391 case 'I': 1392 case 'J': 1393 case 'F': 1394 case 'D': 1395 addToString(c); 1397 return true; 1398 case 'L': 1399 addToString(c); 1401 return jsniMatchQualifiedTypeName('/', ';'); 1402 case '[': 1403 addToString(c); 1405 return jsniMatchParamArrayTypeSignature(); 1406 default: 1407 in.unread(); 1408 reportSyntaxError("msg.jsni.expected.param.type", null); 1409 return false; 1410 } 1411 } 1412 1413 private boolean jsniMatchParamArrayTypeSignature() throws IOException { 1414 return jsniMatchParamTypeSignature(); 1418 } 1419 1420 private boolean jsniMatchMethodSignatureOrFieldName() throws IOException { 1421 int c = in.read(); 1422 1423 if (!Character.isJavaIdentifierStart((char)c)) { 1426 in.unread(); 1427 reportSyntaxError("msg.jsni.expected.identifier", null); 1428 return false; 1429 } 1430 1431 addToString(c); 1432 1433 for (;;) { 1434 c = in.read(); 1435 if (Character.isJavaIdentifierPart((char)c)) { 1436 addToString(c); 1437 } 1438 else if (c == '(') { 1439 addToString(c); 1442 if (jsniMatchParamListSignature()) { 1443 return true; 1447 } 1448 else { 1449 return false; 1452 } 1453 } 1454 else { 1455 in.unread(); 1458 return true; 1459 } 1460 } 1461 } 1462 1463 1470 private boolean jsniMatchQualifiedTypeName(char sepChar, char endChar) 1471 throws IOException { 1472 int c = in.read(); 1473 1474 if (!Character.isJavaIdentifierStart((char)c)) { 1477 in.unread(); 1478 reportSyntaxError("msg.jsni.expected.identifier", null); 1479 return false; 1480 } 1481 1482 addToString(c); 1485 1486 for (;;) { 1489 c = in.read(); 1490 if (Character.isJavaIdentifierPart((char)c)) { 1491 addToString(c); 1492 } 1493 else { 1494 break; 1495 } 1496 } 1497 1498 if (c == sepChar) { 1501 addToString(c); 1502 if (jsniMatchQualifiedTypeName(sepChar, endChar)) { 1503 return true; 1506 } else { 1507 return false; 1511 } 1512 } else if (c == endChar) { 1513 addToString(c); 1516 return true; 1517 } else { 1518 in.unread(); 1521 return true; 1522 } 1523 } 1524 1525 private String getStringFromBuffer() { 1526 return new String (stringBuffer, 0, stringBufferTop); 1527 } 1528 1529 private void addToString(int c) { 1530 if (stringBufferTop == stringBuffer.length) { 1531 char[] tmp = new char[stringBuffer.length * 2]; 1532 System.arraycopy(stringBuffer, 0, tmp, 0, stringBufferTop); 1533 stringBuffer = tmp; 1534 } 1535 stringBuffer[stringBufferTop++] = (char)c; 1536 } 1537 1538 public void reportSyntaxError(String messageProperty, Object [] args) { 1539 String message = Context.getMessage(messageProperty, args); 1540 1541 Context.reportError(message, getSourceName(), 1542 getLineno(), getLine(), getOffset()); 1543 } 1544 1545 private void reportSyntaxWarning(String messageProperty, Object [] args) { 1546 String message = Context.getMessage(messageProperty, args); 1547 Context.reportWarning(message, getSourceName(), 1548 getLineno(), getLine(), getOffset()); 1549 } 1550 1551 public String getSourceName() { return sourceName; } 1552 public int getLineno() { return in.getLineno(); } 1553 public int getOp() { return op; } 1554 public String getString() { return string; } 1555 public double getNumber() { return number; } 1556 public String getLine() { return in.getLine(); } 1557 public int getOffset() { return in.getOffset(); } 1558 public int getTokenno() { return tokenno; } 1559 public boolean eof() { return in.eof(); } 1560 1561 private LineBuffer in; 1563 1564 1565 1569 int flags; 1570 String regExpFlags; 1571 1572 private String sourceName; 1573 private String line; 1574 private int pushbackToken; 1575 private int tokenno; 1576 1577 private int op; 1578 1579 private String string = ""; 1584 private double number; 1585 1586 private char[] stringBuffer = new char[128]; 1587 private int stringBufferTop; 1588} 1589 | Popular Tags |