| 1 2 package org.python.parser; 3 import org.python.parser.ast.modType; 4 5 public class PythonGrammarTokenManager implements PythonGrammarConstants 6 { 7 int indentation[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; 8 int level = 0; 9 int dedents = 0; 10 int parens = 0; 11 int indent; 12 13 boolean expect_indent = false; 14 15 boolean compound = false; 16 17 public boolean single_input = false; 18 19 public boolean partial = false; 21 public boolean stdprompt = false; 24 25 public boolean generator_allowed = false; 26 27 static Token addDedent(Token previous) { 28 Token t = new Token(); 29 t.kind = DEDENT; 30 t.beginLine = previous.beginLine; 31 t.endLine = previous.endLine; 32 t.beginColumn = previous.beginColumn; 33 t.endColumn = previous.endColumn; 34 t.image = "<DEDENT>"; 35 t.specialToken = null; 36 t.next = null; 37 previous.next = t; 38 return t; 39 } 40 41 void CommonTokenAction(Token t) { 42 49 if (t.kind == EOF) { 50 if (!partial || curLexState == MAYBE_FORCE_NEWLINE_IF_EOF) { 52 if (curLexState == DEFAULT) { 53 t.kind = NEWLINE; 54 } 55 else { 56 t.kind = DEDENT; 57 if (level >= 0) level -= 1; 58 } 59 while (level >= 0) { 60 level--; 61 t = addDedent(t); 62 } 63 if (!single_input) { 64 t.kind = EOF; 65 t.image = "<EOF>"; 66 } else { 67 t.kind = NEWLINE; 68 t.image = "<FORCENL>"; 69 single_input = false; 70 } 71 } 72 } else if (t.kind == YIELD) { 73 if (!generator_allowed) { 74 t.kind = NAME; 75 } 76 } 77 } 78 79 void indenting(int ind) { 80 indent = ind; 81 if (indent == indentation[level]) 82 SwitchTo(INDENTATION_UNCHANGED); 83 else 84 SwitchTo(INDENTING); 85 } 86 public java.io.PrintStream debugStream = System.out; 87 public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; } 88 private final int jjMoveStringLiteralDfa0_12() 89 { 90 return 1; 91 } 92 private final int jjStopStringLiteralDfa_10(int pos, long active0, long active1) 93 { 94 switch (pos) 95 { 96 case 0: 97 if ((active1 & 0x80000000000L) != 0L) 98 { 99 jjmatchedKind = 119; 100 return -1; 101 } 102 return -1; 103 case 1: 104 if ((active1 & 0x80000000000L) != 0L) 105 { 106 if (jjmatchedPos == 0) 107 { 108 jjmatchedKind = 119; 109 jjmatchedPos = 0; 110 } 111 return -1; 112 } 113 return -1; 114 default : 115 return -1; 116 } 117 } 118 private final int jjStartNfa_10(int pos, long active0, long active1) 119 { 120 return jjMoveNfa_10(jjStopStringLiteralDfa_10(pos, active0, active1), pos + 1); 121 } 122 private final int jjStopAtPos(int pos, int kind) 123 { 124 jjmatchedKind = kind; 125 jjmatchedPos = pos; 126 return pos + 1; 127 } 128 private final int jjStartNfaWithStates_10(int pos, int kind, int state) 129 { 130 jjmatchedKind = kind; 131 jjmatchedPos = pos; 132 try { curChar = input_stream.readChar(); } 133 catch(java.io.IOException e) { return pos + 1; } 134 return jjMoveNfa_10(state, pos + 1); 135 } 136 private final int jjMoveStringLiteralDfa0_10() 137 { 138 switch(curChar) 139 { 140 case 10: 141 return jjStopAtPos(0, 117); 142 case 13: 143 jjmatchedKind = 118; 144 return jjMoveStringLiteralDfa1_10(0x10000000000000L); 145 case 34: 146 return jjMoveStringLiteralDfa1_10(0x80000000000L); 147 default : 148 return jjMoveNfa_10(0, 0); 149 } 150 } 151 private final int jjMoveStringLiteralDfa1_10(long active1) 152 { 153 try { curChar = input_stream.readChar(); } 154 catch(java.io.IOException e) { 155 jjStopStringLiteralDfa_10(0, 0L, active1); 156 return 1; 157 } 158 switch(curChar) 159 { 160 case 10: 161 if ((active1 & 0x10000000000000L) != 0L) 162 return jjStopAtPos(1, 116); 163 break; 164 case 34: 165 return jjMoveStringLiteralDfa2_10(active1, 0x80000000000L); 166 default : 167 break; 168 } 169 return jjStartNfa_10(0, 0L, active1); 170 } 171 private final int jjMoveStringLiteralDfa2_10(long old1, long active1) 172 { 173 if (((active1 &= old1)) == 0L) 174 return jjStartNfa_10(0, 0L, old1); 175 try { curChar = input_stream.readChar(); } 176 catch(java.io.IOException e) { 177 jjStopStringLiteralDfa_10(1, 0L, active1); 178 return 2; 179 } 180 switch(curChar) 181 { 182 case 34: 183 if ((active1 & 0x80000000000L) != 0L) 184 return jjStopAtPos(2, 107); 185 break; 186 default : 187 break; 188 } 189 return jjStartNfa_10(1, 0L, active1); 190 } 191 private final void jjCheckNAdd(int state) 192 { 193 if (jjrounds[state] != jjround) 194 { 195 jjstateSet[jjnewStateCnt++] = state; 196 jjrounds[state] = jjround; 197 } 198 } 199 private final void jjAddStates(int start, int end) 200 { 201 do { 202 jjstateSet[jjnewStateCnt++] = jjnextStates[start]; 203 } while (start++ != end); 204 } 205 private final void jjCheckNAddTwoStates(int state1, int state2) 206 { 207 jjCheckNAdd(state1); 208 jjCheckNAdd(state2); 209 } 210 private final void jjCheckNAddStates(int start, int end) 211 { 212 do { 213 jjCheckNAdd(jjnextStates[start]); 214 } while (start++ != end); 215 } 216 private final void jjCheckNAddStates(int start) 217 { 218 jjCheckNAdd(jjnextStates[start]); 219 jjCheckNAdd(jjnextStates[start + 1]); 220 } 221 static final long[] jjbitVec0 = { 222 0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL 223 }; 224 static final long[] jjbitVec2 = { 225 0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL 226 }; 227 private final int jjMoveNfa_10(int startState, int curPos) 228 { 229 int[] nextStates; 230 int startsAt = 0; 231 jjnewStateCnt = 3; 232 int i = 1; 233 jjstateSet[0] = startState; 234 int j, kind = 0x7fffffff; 235 for (;;) 236 { 237 if (++jjround == 0x7fffffff) 238 ReInitRounds(); 239 if (curChar < 64) 240 { 241 long l = 1L << curChar; 242 MatchLoop: do 243 { 244 switch(jjstateSet[--i]) 245 { 246 case 0: 247 if ((0xffffffffffffdbffL & l) != 0L && kind > 119) 248 kind = 119; 249 break; 250 case 2: 251 if ((0xffffffffffffdbffL & l) != 0L && kind > 120) 252 kind = 120; 253 break; 254 default : break; 255 } 256 } while(i != startsAt); 257 } 258 else if (curChar < 128) 259 { 260 long l = 1L << (curChar & 077); 261 MatchLoop: do 262 { 263 switch(jjstateSet[--i]) 264 { 265 case 0: 266 if (kind > 119) 267 kind = 119; 268 if (curChar == 92) 269 jjstateSet[jjnewStateCnt++] = 2; 270 break; 271 case 1: 272 if (curChar == 92) 273 jjstateSet[jjnewStateCnt++] = 2; 274 break; 275 case 2: 276 if (kind > 120) 277 kind = 120; 278 break; 279 default : break; 280 } 281 } while(i != startsAt); 282 } 283 else 284 { 285 int hiByte = (int)(curChar >> 8); 286 int i1 = hiByte >> 6; 287 long l1 = 1L << (hiByte & 077); 288 int i2 = (curChar & 0xff) >> 6; 289 long l2 = 1L << (curChar & 077); 290 MatchLoop: do 291 { 292 switch(jjstateSet[--i]) 293 { 294 case 0: 295 if (jjCanMove_0(hiByte, i1, i2, l1, l2) && kind > 119) 296 kind = 119; 297 break; 298 case 2: 299 if (jjCanMove_0(hiByte, i1, i2, l1, l2) && kind > 120) 300 kind = 120; 301 break; 302 default : break; 303 } 304 } while(i != startsAt); 305 } 306 if (kind != 0x7fffffff) 307 { 308 jjmatchedKind = kind; 309 jjmatchedPos = curPos; 310 kind = 0x7fffffff; 311 } 312 ++curPos; 313 if ((i = jjnewStateCnt) == (startsAt = 3 - (jjnewStateCnt = startsAt))) 314 return curPos; 315 try { curChar = input_stream.readChar(); } 316 catch(java.io.IOException e) { return curPos; } 317 } 318 } 319 private final int jjMoveStringLiteralDfa0_3() 320 { 321 return 1; 322 } 323 private final int jjStopStringLiteralDfa_8(int pos, long active0, long active1) 324 { 325 switch (pos) 326 { 327 case 0: 328 if ((active1 & 0x400000000000L) != 0L) 329 { 330 jjmatchedKind = 115; 331 return 2; 332 } 333 return -1; 334 case 1: 335 if ((active1 & 0x400000000000L) != 0L) 336 { 337 jjmatchedKind = 111; 338 jjmatchedPos = 1; 339 return -1; 340 } 341 return -1; 342 default : 343 return -1; 344 } 345 } 346 private final int jjStartNfa_8(int pos, long active0, long active1) 347 { 348 return jjMoveNfa_8(jjStopStringLiteralDfa_8(pos, active0, active1), pos + 1); 349 } 350 private final int jjStartNfaWithStates_8(int pos, int kind, int state) 351 { 352 jjmatchedKind = kind; 353 jjmatchedPos = pos; 354 try { curChar = input_stream.readChar(); } 355 catch(java.io.IOException e) { return pos + 1; } 356 return jjMoveNfa_8(state, pos + 1); 357 } 358 private final int jjMoveStringLiteralDfa0_8() 359 { 360 switch(curChar) 361 { 362 case 34: 363 return jjStopAtPos(0, 105); 364 case 92: 365 return jjMoveStringLiteralDfa1_8(0x400000000000L); 366 default : 367 return jjMoveNfa_8(0, 0); 368 } 369 } 370 private final int jjMoveStringLiteralDfa1_8(long active1) 371 { 372 try { curChar = input_stream.readChar(); } 373 catch(java.io.IOException e) { 374 jjStopStringLiteralDfa_8(0, 0L, active1); 375 return 1; 376 } 377 switch(curChar) 378 { 379 case 13: 380 return jjMoveStringLiteralDfa2_8(active1, 0x400000000000L); 381 default : 382 break; 383 } 384 return jjStartNfa_8(0, 0L, active1); 385 } 386 private final int jjMoveStringLiteralDfa2_8(long old1, long active1) 387 { 388 if (((active1 &= old1)) == 0L) 389 return jjStartNfa_8(0, 0L, old1); 390 try { curChar = input_stream.readChar(); } 391 catch(java.io.IOException e) { 392 jjStopStringLiteralDfa_8(1, 0L, active1); 393 return 2; 394 } 395 switch(curChar) 396 { 397 case 10: 398 if ((active1 & 0x400000000000L) != 0L) 399 return jjStopAtPos(2, 110); 400 break; 401 default : 402 break; 403 } 404 return jjStartNfa_8(1, 0L, active1); 405 } 406 private final int jjMoveNfa_8(int startState, int curPos) 407 { 408 int[] nextStates; 409 int startsAt = 0; 410 jjnewStateCnt = 4; 411 int i = 1; 412 jjstateSet[0] = startState; 413 int j, kind = 0x7fffffff; 414 for (;;) 415 { 416 if (++jjround == 0x7fffffff) 417 ReInitRounds(); 418 if (curChar < 64) 419 { 420 long l = 1L << curChar; 421 MatchLoop: do 422 { 423 switch(jjstateSet[--i]) 424 { 425 case 0: 426 if ((0xffffffffffffdbffL & l) != 0L && kind > 115) 427 kind = 115; 428 break; 429 case 2: 430 if ((0x2400L & l) != 0L) 431 { 432 if (kind > 111) 433 kind = 111; 434 } 435 else if (curChar == 34) 436 { 437 if (kind > 115) 438 kind = 115; 439 } 440 break; 441 case 3: 442 if (curChar == 34 && kind > 115) 443 kind = 115; 444 break; 445 default : break; 446 } 447 } while(i != startsAt); 448 } 449 else if (curChar < 128) 450 { 451 long l = 1L << (curChar & 077); 452 MatchLoop: do 453 { 454 switch(jjstateSet[--i]) 455 { 456 case 0: 457 if (kind > 115) 458 kind = 115; 459 if (curChar == 92) 460 jjAddStates(0, 1); 461 break; 462 case 2: 463 if (curChar == 92 && kind > 115) 464 kind = 115; 465 break; 466 case 1: 467 if (curChar == 92) 468 jjAddStates(0, 1); 469 break; 470 default : break; 471 } 472 } while(i != startsAt); 473 } 474 else 475 { 476 int hiByte = (int)(curChar >> 8); 477 int i1 = hiByte >> 6; 478 long l1 = 1L << (hiByte & 077); 479 int i2 = (curChar & 0xff) >> 6; 480 long l2 = 1L << (curChar & 077); 481 MatchLoop: do 482 { 483 switch(jjstateSet[--i]) 484 { 485 case 0: 486 if (jjCanMove_0(hiByte, i1, i2, l1, l2) && kind > 115) 487 kind = 115; 488 break; 489 default : break; 490 } 491 } while(i != startsAt); 492 } 493 if (kind != 0x7fffffff) 494 { 495 jjmatchedKind = kind; 496 jjmatchedPos = curPos; 497 kind = 0x7fffffff; 498 } 499 ++curPos; 500 if ((i = jjnewStateCnt) == (startsAt = 4 - (jjnewStateCnt = startsAt))) 501 return curPos; 502 try { curChar = input_stream.readChar(); } 503 catch(java.io.IOException e) { return curPos; } 504 } 505 } 506 private final int jjMoveStringLiteralDfa0_2() 507 { 508 return jjMoveNfa_2(0, 0); 509 } 510 private final int jjMoveNfa_2(int startState, int curPos) 511 { 512 int[] nextStates; 513 int startsAt = 0; 514 jjnewStateCnt = 1; 515 int i = 1; 516 jjstateSet[0] = startState; 517 int j, kind = 0x7fffffff; 518 for (;;) 519 { 520 if (++jjround == 0x7fffffff) 521 ReInitRounds(); 522 if (curChar < 64) 523 { 524 long l = 1L << curChar; 525 MatchLoop: do 526 { 527 switch(jjstateSet[--i]) 528 { 529 case 0: 530 if ((0x2400L & l) != 0L) 531 kind = 7; 532 break; 533 default : break; 534 } 535 } while(i != startsAt); 536 } 537 else if (curChar < 128) 538 { 539 long l = 1L << (curChar & 077); 540 MatchLoop: do 541 { 542 switch(jjstateSet[--i]) 543 { 544 default : break; 545 } 546 } while(i != startsAt); 547 } 548 else 549 { 550 int hiByte = (int)(curChar >> 8); 551 int i1 = hiByte >> 6; 552 long l1 = 1L << (hiByte & 077); 553 int i2 = (curChar & 0xff) >> 6; 554 long l2 = 1L << (curChar & 077); 555 MatchLoop: do 556 { 557 switch(jjstateSet[--i]) 558 { 559 default : break; 560 } 561 } while(i != startsAt); 562 } 563 if (kind != 0x7fffffff) 564 { 565 jjmatchedKind = kind; 566 jjmatchedPos = curPos; 567 kind = 0x7fffffff; 568 } 569 ++curPos; 570 if ((i = jjnewStateCnt) == (startsAt = 1 - (jjnewStateCnt = startsAt))) 571 return curPos; 572 try { curChar = input_stream.readChar(); } 573 catch(java.io.IOException e) { return curPos; } 574 } 575 } 576 private final int jjMoveStringLiteralDfa0_1() 577 { 578 return jjMoveNfa_1(0, 0); 579 } 580 private final int jjMoveNfa_1(int startState, int curPos) 581 { 582 int[] nextStates; 583 int startsAt = 0; 584 jjnewStateCnt = 1; 585 int i = 1; 586 jjstateSet[0] = startState; 587 int j, kind = 0x7fffffff; 588 for (;;) 589 { 590 if (++jjround == 0x7fffffff) 591 ReInitRounds(); 592 if (curChar < 64) 593 { 594 long l = 1L << curChar; 595 MatchLoop: do 596 { 597 switch(jjstateSet[--i]) 598 { 599 case 0: 600 if ((0x2400L & l) != 0L) 601 kind = 6; 602 break; 603 default : break; 604 } 605 } while(i != startsAt); 606 } 607 else if (curChar < 128) 608 { 609 long l = 1L << (curChar & 077); 610 MatchLoop: do 611 { 612 switch(jjstateSet[--i]) 613 { 614 default : break; 615 } 616 } while(i != startsAt); 617 } 618 else 619 { 620 int hiByte = (int)(curChar >> 8); 621 int i1 = hiByte >> 6; 622 long l1 = 1L << (hiByte & 077); 623 int i2 = (curChar & 0xff) >> 6; 624 long l2 = 1L << (curChar & 077); 625 MatchLoop: do 626 { 627 switch(jjstateSet[--i]) 628 { 629 default : break; 630 } 631 } while(i != startsAt); 632 } 633 if (kind != 0x7fffffff) 634 { 635 jjmatchedKind = kind; 636 jjmatchedPos = curPos; 637 kind = 0x7fffffff; 638 } 639 ++curPos; 640 if ((i = jjnewStateCnt) == (startsAt = 1 - (jjnewStateCnt = startsAt))) 641 return curPos; 642 try { curChar = input_stream.readChar(); } 643 catch(java.io.IOException e) { return curPos; } 644 } 645 } 646 private final int jjMoveStringLiteralDfa0_11() 647 { 648 return 1; 649 } 650 private final int jjMoveStringLiteralDfa0_6() 651 { 652 switch(curChar) 653 { 654 case 60: 655 return jjMoveStringLiteralDfa1_6(0x8000L); 656 default : 657 return 1; 658 } 659 } 660 private final int jjMoveStringLiteralDfa1_6(long active0) 661 { 662 try { curChar = input_stream.readChar(); } 663 catch(java.io.IOException e) { 664 return 1; 665 } 666 switch(curChar) 667 { 668 case 73: 669 return jjMoveStringLiteralDfa2_6(active0, 0x8000L); 670 default : 671 return 2; 672 } 673 } 674 private final int jjMoveStringLiteralDfa2_6(long old0, long active0) 675 { 676 if (((active0 &= old0)) == 0L) 677 return 2; 678 try { curChar = input_stream.readChar(); } 679 catch(java.io.IOException e) { 680 return 2; 681 } 682 switch(curChar) 683 { 684 case 78: 685 return jjMoveStringLiteralDfa3_6(active0, 0x8000L); 686 default : 687 return 3; 688 } 689 } 690 private final int jjMoveStringLiteralDfa3_6(long old0, long active0) 691 { 692 if (((active0 &= old0)) == 0L) 693 return 3; 694 try { curChar = input_stream.readChar(); } 695 catch(java.io.IOException e) { 696 return 3; 697 } 698 switch(curChar) 699 { 700 case 68: 701 return jjMoveStringLiteralDfa4_6(active0, 0x8000L); 702 default : 703 return 4; 704 } 705 } 706 private final int jjMoveStringLiteralDfa4_6(long old0, long active0) 707 { 708 if (((active0 &= old0)) == 0L) 709 return 4; 710 try { curChar = input_stream.readChar(); } 711 catch(java.io.IOException e) { 712 return 4; 713 } 714 switch(curChar) 715 { 716 case 69: 717 return jjMoveStringLiteralDfa5_6(active0, 0x8000L); 718 default : 719 return 5; 720 } 721 } 722 private final int jjMoveStringLiteralDfa5_6(long old0, long active0) 723 { 724 if (((active0 &= old0)) == 0L) 725 return 5; 726 try { curChar = input_stream.readChar(); } 727 catch(java.io.IOException e) { 728 return 5; 729 } 730 switch(curChar) 731 { 732 case 78: 733 return jjMoveStringLiteralDfa6_6(active0, 0x8000L); 734 default : 735 return 6; 736 } 737 } 738 private final int jjMoveStringLiteralDfa6_6(long old0, long active0) 739 { 740 if (((active0 &= old0)) == 0L) 741 return 6; 742 try { curChar = input_stream.readChar(); } 743 catch(java.io.IOException e) { 744 return 6; 745 } 746 switch(curChar) 747 { 748 case 84: 749 return jjMoveStringLiteralDfa7_6(active0, 0x8000L); 750 default : 751 return 7; 752 } 753 } 754 private final int jjMoveStringLiteralDfa7_6(long old0, long active0) 755 { 756 if (((active0 &= old0)) == 0L) 757 return 7; 758 try { curChar = input_stream.readChar(); } 759 catch(java.io.IOException e) { 760 return 7; 761 } 762 switch(curChar) 763 { 764 case 62: 765 if ((active0 & 0x8000L) != 0L) 766 return jjStopAtPos(7, 15); 767 break; 768 default : 769 return 8; 770 } 771 return 8; 772 } 773 private final int jjStopStringLiteralDfa_9(int pos, long active0, long active1) 774 { 775 switch (pos) 776 { 777 case 0: 778 if ((active1 & 0x40000000000L) != 0L) 779 { 780 jjmatchedKind = 119; 781 return -1; 782 } 783 return -1; 784 case 1: 785 if ((active1 & 0x40000000000L) != 0L) 786 { 787 if (jjmatchedPos == 0) 788 { 789 jjmatchedKind = 119; 790 jjmatchedPos = 0; 791 } 792 return -1; 793 } 794 return -1; 795 default : 796 return -1; 797 } 798 } 799 private final int jjStartNfa_9(int pos, long active0, long active1) 800 { 801 return jjMoveNfa_9(jjStopStringLiteralDfa_9(pos, active0, active1), pos + 1); 802 } 803 private final int jjStartNfaWithStates_9(int pos, int kind, int state) 804 { 805 jjmatchedKind = kind; 806 jjmatchedPos = pos; 807 try { curChar = input_stream.readChar(); } 808 catch(java.io.IOException e) { return pos + 1; } 809 return jjMoveNfa_9(state, pos + 1); 810 } 811 private final int jjMoveStringLiteralDfa0_9() 812 { 813 switch(curChar) 814 { 815 case 10: 816 return jjStopAtPos(0, 117); 817 case 13: 818 jjmatchedKind = 118; 819 return jjMoveStringLiteralDfa1_9(0x10000000000000L); 820 case 39: 821 return jjMoveStringLiteralDfa1_9(0x40000000000L); 822 default : 823 return jjMoveNfa_9(0, 0); 824 } 825 } 826 private final int jjMoveStringLiteralDfa1_9(long active1) 827 { 828 try { curChar = input_stream.readChar(); } 829 catch(java.io.IOException e) { 830 jjStopStringLiteralDfa_9(0, 0L, active1); 831 return 1; 832 } 833 switch(curChar) 834 { 835 case 10: 836 if ((active1 & 0x10000000000000L) != 0L) 837 return jjStopAtPos(1, 116); 838 break; 839 case 39: 840 return jjMoveStringLiteralDfa2_9(active1, 0x40000000000L); 841 default : 842 break; 843 } 844 return jjStartNfa_9(0, 0L, active1); 845 } 846 private final int jjMoveStringLiteralDfa2_9(long old1, long active1) 847 { 848 if (((active1 &= old1)) == 0L) 849 return jjStartNfa_9(0, 0L, old1); 850 try { curChar = input_stream.readChar(); } 851 catch(java.io.IOException e) { 852 jjStopStringLiteralDfa_9(1, 0L, active1); 853 return 2; 854 } 855 switch(curChar) 856 { 857 case 39: 858 if ((active1 & 0x40000000000L) != 0L) 859 return jjStopAtPos(2, 106); 860 break; 861 default : 862 break; 863 } 864 return jjStartNfa_9(1, 0L, active1); 865 } 866 private final int jjMoveNfa_9(int startState, int curPos) 867 { 868 int[] nextStates; 869 int startsAt = 0; 870 jjnewStateCnt = |