1 package fri.patterns.interpreter.parsergenerator.lexer; 2 3 import java.util.*; 4 import java.io.IOException ; 5 import java.io.Serializable ; 6 import fri.patterns.interpreter.parsergenerator.Token; 7 import fri.patterns.interpreter.parsergenerator.syntax.Rule; 8 9 21 22 class Consumer implements 23 Comparable , 24 Serializable 25 { 26 private List sequence = new ArrayList(); 27 private List constraints; 28 private boolean nullable = false; 29 private boolean repeatable = false; 30 protected Rule rule; 31 protected int fixedLength = -1, startLength = -1, variance = -1; 32 33 34 35 Consumer(Rule rule) { 36 this.rule = rule; 37 } 38 39 40 Consumer(String charOrString) { 41 append(charOrString); 42 } 43 44 45 protected Consumer() { 46 } 47 48 49 50 public void append(String charOrString) { 51 Object o = sequence.size() > 0 ? sequence.get(sequence.size() - 1) : null; if (o instanceof String ) { String s = (String )o; 54 s = s + charOrString; sequence.set(sequence.size() - 1, s); 56 } 57 else { 58 sequence.add(charOrString); 59 } 60 } 61 62 63 public void append(Reference subConsumer) { 64 sequence.add(subConsumer); 65 } 66 67 68 public void append(Consumer subConsumer) { 69 sequence.add(subConsumer); 70 } 71 72 73 public void appendSet(String high) 74 throws LexerException 75 { 76 String low = (String ) sequence.get(sequence.size() - 1); if (low.length() > 1) { int i = low.length() - 1; 79 sequence.set(sequence.size() - 1, low.substring(0, i)); sequence.add(new CharacterSet(""+low.charAt(i), high)); } 82 else { 83 sequence.set(sequence.size() - 1, new CharacterSet(low, high)); 84 } 85 } 86 87 88 public void subtract(Consumer constraint) { 89 if (constraints == null) 90 constraints = new ArrayList(); 91 constraints.add(constraint); 92 } 93 94 95 public void subtract(Reference constraint) { 96 if (constraints == null) 97 constraints = new ArrayList(); 98 constraints.add(constraint); 99 } 100 101 102 103 public void resolveConsumerReferences(Map charConsumers, Map doneList) 104 throws LexerException 105 { 106 if (doneList.get(this) != null) 107 return; 108 doneList.put(this, this); 109 110 List [] varr = new List[] { sequence, getAlternatives(), constraints }; 111 for (int j = 0; j < varr.length; j++) { 112 List v = varr[j]; 113 114 if (v != null) { 115 for (int i = 0; v != null && i < v.size(); i++) { 116 Object o = v.get(i); 117 118 if (o instanceof Reference) { 119 Reference cr = (Reference)o; 121 Object cc = charConsumers.get(cr.nonterminal); 122 123 if (cc == null) 124 throw new LexerException("Consumer-Reference not found: "+cr.nonterminal); 125 126 v.set(i, cc); 127 } 128 else 129 if (o instanceof Consumer) { 130 Consumer cc = (Consumer)o; 131 cc.resolveConsumerReferences(charConsumers, doneList); 132 } 133 } 134 } 135 } 136 } 137 138 139 public void setNullable() { 140 nullable = true; 141 } 142 143 public boolean isNullable() { 144 return nullable; 145 } 146 147 public void setRepeatable() { 148 repeatable = true; 149 } 150 151 public boolean isRepeatable() { 152 return repeatable; 153 } 154 155 156 public List getAlternatives() { 157 return null; 158 } 159 160 161 162 public int compareTo(Object o) { 163 Consumer cc = (Consumer)o; 164 int i; 165 i = getStartVariance() - cc.getStartVariance(); if (i != 0) 167 return i; 168 i = cc.getStartLength() - getStartLength(); if (i != 0) 170 return i; 171 i = cc.getFixedLength() - getFixedLength(); if (i != 0) 173 return i; 174 return 0; 175 } 176 177 178 public Character getStartCharacter() { 179 Object o = sequence.get(0); 180 if (o instanceof Consumer) 182 return ((Consumer)o).getStartCharacter(); 183 else 184 if (o instanceof CharacterSet) 185 return null; 186 else 187 return new Character (((String )o).charAt(0)); 188 } 189 190 191 192 public int getStartVariance() { 193 if (this.variance > 0) 194 return this.variance; 195 196 if (getStartCharacter() != null) 197 return this.variance = 1; 198 199 int variance; 200 Object o = sequence.get(0); 201 if (o instanceof Consumer) 202 variance = ((Consumer)o).getStartVariance(); 203 else 204 if (o instanceof CharacterSet) 205 variance = ((CharacterSet)o).getVariance(); 206 else 207 throw new IllegalStateException ("No fixed start character, no character set, where am i?"); 208 209 for (int i = 0; constraints != null && i < constraints.size(); i++) 210 variance -= ((Consumer) constraints.get(i)).getStartVariance(); 211 212 return this.variance = variance; 213 } 214 215 216 public int getFixedLength() { 217 if (fixedLength >= 0) return fixedLength; 219 fixedLength = getSomeLength(false, new ArrayList()); 220 return fixedLength; 221 } 222 223 224 public int getStartLength() { 225 if (startLength >= 0) return startLength; 227 List reason = new ArrayList(); 228 startLength = getSomeLength(true, reason); 229 return startLength; 231 } 232 233 234 protected int getSomeLength(boolean exploreStartLength, List breakIndicator) { 235 int len = 0; 236 237 for (int i = 0; breakIndicator.size() <= 0 && i < sequence.size(); i++) { 238 Object o = sequence.get(i); 239 240 if (o instanceof Consumer) { 241 Consumer cc = (Consumer)o; 242 243 if (cc.isNullable()) { breakIndicator.add("nullable"); return len; 246 } 247 else 248 if (cc.isRepeatable()) { len += cc.getSomeLength(exploreStartLength, breakIndicator); 250 breakIndicator.add("repeatable"); return len; 252 } 253 254 len += cc.getSomeLength(exploreStartLength, breakIndicator); 255 } 256 else 257 if (o instanceof CharacterSet) { 258 if (exploreStartLength) { 259 breakIndicator.add("set"); return len; 261 } 262 len += 1; } 264 else { len += ((String )o).length(); } 267 } 268 269 return len; 270 } 271 272 273 274 275 Consumer optimize() { 276 if (constraints == null && sequence.size() == 1 && sequence.get(0) instanceof Consumer) { 277 Consumer cc = (Consumer) sequence.get(0); 279 cc.rule = rule; 280 return cc; 281 } 282 return this; 283 } 284 285 289 boolean matchesRepeatableRule(Rule rule) { 290 if (rule.rightSize() != this.rule.rightSize() + 1) 291 return false; 292 for (int i = 0; i < this.rule.rightSize(); i++) 293 if (this.rule.getRightSymbol(i).equals(rule.getRightSymbol(i + 1)) == false) 294 return false; 295 return true; 296 } 297 298 299 300 302 303 public void setStrategyFactoryMethod(StrategyFactoryMethod strategyFactoryMethod) { 304 for (int i = 0; i < sequence.size(); i++) { 305 Object c = sequence.get(i); 306 if (c instanceof Consumer) { 307 ((Consumer) c).setStrategyFactoryMethod(strategyFactoryMethod); 308 } 309 } 310 } 311 312 318 public ResultTree consume(InputText input) 319 throws IOException 320 { 321 ResultTree result = isRepeatable() ? ensureResultTree(null) : null; ResultTree r; 324 Token.Address start = new Token.Address(input.getScanLine(), input.getScanColumn(), input.getScanOffset()); 325 326 do { 328 r = consumeInternal(input); 329 if (r != null && r.hasText()) 330 result = (result == null) ? r : result.addChild(r); 331 } 332 while (r != null && isRepeatable()); 333 334 if (result != null && isRepeatable() && result.getChildCount() <= 0) 336 result = null; 337 338 if (result == null && isNullable()) result = ensureResultTree(null); 341 if (result != null) { 343 Token.Address end = new Token.Address(input.getScanLine(), input.getScanColumn(), input.getScanOffset()); 344 result.setRange(new Token.Range(start, end)); 345 } 346 347 return result; 348 } 349 350 351 355 protected ResultTree consumeInternal(InputText input) 356 throws IOException 357 { 358 ResultTree result = null; 359 boolean ok = true; 360 int mark = input.getMark(); 361 Token.Address start = new Token.Address(input.getScanLine(), input.getScanColumn(), input.getScanOffset()); 362 363 for (int i = 0; ok && i < sequence.size(); i++) { 365 366 for (int j = 0; ok && constraints != null && j < constraints.size(); j++) { 368 Consumer cc = (Consumer) constraints.get(j); 369 370 int mark1 = input.getMark(); ok = (cc.consumeInternal(input) == null); input.setMark(mark1); } 374 375 if (ok) { Object o = sequence.get(i); 377 378 if (o instanceof Consumer) { Consumer cc = (Consumer)o; 380 ResultTree r = cc.consume(input); 381 382 if (r == null) 383 ok = false; 384 else 385 result = ensureResultTree(result).addChild(r); 386 } 387 else 388 if (o instanceof CharacterSet) { CharacterSet charSet = (CharacterSet)o; 390 int ic = input.read(); 391 char c = (char)ic; 392 if (ic == Input.EOF || charSet.includes(c) == false) 393 ok = false; 394 else 395 result = ensureResultTree(result).append(c); 396 } 397 else { String s = (String )o; 399 int j = 0; 400 401 do { 402 int ic = input.read(); 403 ok = (ic != Input.EOF && ((char)ic) == s.charAt(j)); 404 j++; 405 } 406 while (ok && j < s.length()); 407 408 if (ok) 409 result = ensureResultTree(result).append(s); 410 411 } 413 } } 416 417 if (ok && result != null && result.hasText()) { Token.Address end = new Token.Address(input.getScanLine(), input.getScanColumn(), input.getScanOffset()); 419 result.setRange(new Token.Range(start, end)); 420 return result; } 423 424 input.setMark(mark); 426 return null; } 429 430 431 private ResultTree ensureResultTree(ResultTree result) { 432 if (result == null) 433 result = new ResultTree(rule); 434 return result; 435 } 436 437 438 439 440 public String toString() { 441 return hashCode()+ 442 "("+getStartVariance()+","+getStartLength()+","+getFixedLength()+")> "+ 443 toStringBase()+ 444 (isNullable() && isRepeatable() ? " *" : isNullable() ? " ?" : isRepeatable() ? " +" : ""); 445 } 446 447 448 protected String toStringBase() { 449 StringBuffer sb = new StringBuffer (); 450 listToString(sequence, sb, " ", false); 451 listToString(constraints, sb, " - ", true); 452 return sb.toString(); 453 } 454 455 456 protected void listToString(List list, StringBuffer sb, String separator, boolean separatorAtFirst) { 457 for (int i = 0; list != null && i < list.size(); i++) { 458 Object o = list.get(i); 459 460 if (separatorAtFirst || i > 0) 461 sb.append(separator); 462 463 if (o instanceof Consumer) 464 sb.append("["+o.hashCode()+"]"); 465 else sb.append(o.toString()); 467 } 468 } 469 470 471 472 476 public boolean overlaps(Consumer cc) { 477 Object o1 = sequence.get(0); 478 479 if (o1 instanceof Consumer) { return ((Consumer)o1).overlaps(cc); 481 } 482 else { if (cc.sequence.size() <= 0) return cc.overlaps(this); 485 486 Object o2 = cc.sequence.get(0); 487 488 if (o2 instanceof Consumer) { return ((Consumer)o2).overlaps(this); 490 } 491 else 492 if (o1 instanceof CharacterSet) { 493 if (o2 instanceof CharacterSet) 494 return ((CharacterSet)o1).overlaps((CharacterSet)o2); 495 else 496 if (o2 instanceof String ) 497 return ((CharacterSet)o1).includes(((String )o2).charAt(0)); 498 } 499 else 500 if (o2 instanceof CharacterSet) { 501 if (o1 instanceof String ) 502 return ((CharacterSet)o2).includes(((String )o1).charAt(0)); 503 } 504 505 String seq1 = (String ) o1; 506 String seq2 = (String ) o2; 507 return seq1.charAt(0) == seq2.charAt(0); 508 } 509 } 510 511 512 513 514 516 517 518 public static class Reference 519 { 520 String nonterminal; 521 522 Reference(String nonterminal) { 523 this.nonterminal = nonterminal; 524 } 525 public String toString() { 526 return nonterminal; 527 } 528 } 529 530 531 532 private static class CharacterSet implements Serializable 533 { 534 private String stringRepres; 535 private char firstChar, lastChar; 536 537 CharacterSet(String first, String last) 538 throws LexerException 539 { 540 this.firstChar = first.charAt(0); 541 this.lastChar = last.charAt(0); 542 543 if (firstChar >= lastChar) 544 throw new LexerException("First character is bigger equal last: "+toString()); 545 } 546 547 public char getFirstChar() { 548 return firstChar; 549 } 550 551 public char getLastChar() { 552 return lastChar; 553 } 554 555 556 public int getVariance() { 557 return lastChar - firstChar; 558 } 559 560 561 public boolean includes(char c) { 562 return c >= firstChar && c <= lastChar; 563 } 564 565 566 public boolean overlaps(CharacterSet other) { 567 return other.includes(firstChar) || other.includes(lastChar) || includes(other.firstChar) || includes(other.lastChar); 568 } 569 570 public String toString() { 571 if (stringRepres == null) { 572 if (Character.isISOControl(firstChar) || Character.isISOControl(lastChar)) 573 this.stringRepres = Integer.toHexString(firstChar)+".."+Integer.toHexString(lastChar); 574 else 575 this.stringRepres = firstChar+".."+lastChar; 576 } 577 return stringRepres; 578 } 579 } 580 581 } | Popular Tags |