1 4 package com.tc.objectserver.lockmanager.impl; 5 6 import com.tc.async.api.Sink; 7 import com.tc.async.impl.MockSink; 8 import com.tc.net.protocol.tcm.ChannelID; 9 import com.tc.object.lockmanager.api.LockID; 10 import com.tc.object.lockmanager.api.LockLevel; 11 import com.tc.object.lockmanager.api.ThreadID; 12 import com.tc.object.lockmanager.api.WaitTimer; 13 import com.tc.object.lockmanager.impl.WaitTimerImpl; 14 import com.tc.object.tx.WaitInvocation; 15 import com.tc.objectserver.lockmanager.api.LockAwardContext; 16 import com.tc.objectserver.lockmanager.api.LockEventListener; 17 import com.tc.objectserver.lockmanager.api.LockEventMonitor; 18 import com.tc.objectserver.lockmanager.api.NotifiedWaiters; 19 import com.tc.objectserver.lockmanager.api.NullChannelManager; 20 import com.tc.objectserver.lockmanager.api.TCIllegalMonitorStateException; 21 import com.tc.objectserver.lockmanager.api.LockEventMonitor.CallContext; 22 import com.tc.util.TCAssertionError; 23 import com.tc.util.concurrent.ThreadUtil; 24 25 import java.util.Collection ; 26 import java.util.Iterator ; 27 28 import junit.framework.TestCase; 29 30 public class LockTest extends TestCase { 31 32 private Sink sink; 33 private long uniqueId = 100000L; 34 private WaitTimer waitTimer; 35 private LockManagerImpl lockMgr = new LockManagerImpl(new NullChannelManager()); 36 private NotifiedWaiters notifiedWaiters; 37 38 protected void setUp() throws Exception { 39 super.setUp(); 40 this.notifiedWaiters = new NotifiedWaiters(); 41 this.sink = new MockSink(); 42 this.waitTimer = new WaitTimerImpl(); 43 } 44 45 public void testLockClear() { 46 49 } 51 52 public void testUpgrade() throws Exception { 53 ChannelID channelId1 = new ChannelID(1); 54 ThreadID txnId1 = new ThreadID(1); 55 ThreadID txnId2 = new ThreadID(2); 56 ThreadID txnId3 = new ThreadID(3); 57 58 ServerThreadContext thread1 = makeTxn(channelId1, txnId1); 59 ServerThreadContext thread2 = makeTxn(channelId1, txnId2); 60 ServerThreadContext thread3 = makeTxn(channelId1, txnId3); 61 62 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 63 lock.requestLock(thread1, LockLevel.READ, sink); 64 lock.requestLock(thread2, LockLevel.READ, sink); 65 lock.requestLock(thread3, LockLevel.READ, sink); 66 67 assertEquals(3, lock.getHoldersCount()); 69 assertEquals(0, lock.getPendingUpgradeCount()); 70 lock.requestLock(thread1, LockLevel.WRITE, sink); 71 assertEquals(1, lock.getPendingUpgradeCount()); 72 assertEquals(3, lock.getHoldersCount()); 73 74 lock.removeCurrentHold(thread2); 76 lock.nextPending(); 77 assertEquals(1, lock.getPendingUpgradeCount()); 78 assertEquals(2, lock.getHoldersCount()); 79 80 lock.removeCurrentHold(thread3); 82 lock.nextPending(); 83 84 assertEquals(0, lock.getPendingUpgradeCount()); 86 assertEquals(1, lock.getHoldersCount()); 87 Holder holder = (Holder) lock.getHoldersCollection().toArray()[0]; 88 assertEquals(channelId1, holder.getChannelID()); 89 assertEquals(txnId1, holder.getThreadID()); 90 assertTrue(holder.isUpgrade()); 91 92 lock.requestLock(thread2, LockLevel.READ, sink); 94 lock.requestLock(thread3, LockLevel.WRITE, sink); 95 assertEquals(2, lock.getPendingCount()); 96 97 lock.removeCurrentHold(thread1); 99 lock.awardAllReads(); 100 assertEquals(1, lock.getPendingCount()); 101 Holder[] holders = (Holder[]) lock.getHoldersCollection().toArray(new Holder[] {}); 102 assertEquals(2, holders.length); 103 boolean tx1 = false, tx2 = false; 104 105 for (int i = 0; i < 2; i++) { 106 Holder h = holders[i]; 107 assertEquals(channelId1, h.getChannelID()); 108 if (h.getThreadID().equals(txnId1)) { 109 assertFalse(tx1); 110 tx1 = true; 111 } else if (h.getThreadID().equals(txnId2)) { 112 assertFalse(tx2); 113 tx2 = true; 114 } else { 115 fail(h.getThreadID().toString()); 116 } 117 118 assertEquals(LockLevel.READ, h.getLockLevel()); 119 } 120 assertTrue(tx1); 121 assertTrue(tx2); 122 123 lock.removeCurrentHold(thread1); 125 lock.nextPending(); 126 assertEquals(1, lock.getHoldersCount()); 127 assertEquals(1, lock.getPendingCount()); 128 129 lock.removeCurrentHold(thread2); 131 lock.nextPending(); 132 assertEquals(0, lock.getPendingCount()); 133 assertEquals(1, lock.getHoldersCount()); 134 holder = (Holder) lock.getHoldersCollection().toArray()[0]; 135 assertEquals(channelId1, holder.getChannelID()); 136 assertEquals(txnId3, holder.getThreadID()); 137 assertEquals(LockLevel.WRITE, holder.getLockLevel()); 138 } 139 140 private static ServerThreadContext makeTxn(ChannelID cid, ThreadID threadID) { 141 return new ServerThreadContext(cid, threadID); 142 } 143 144 public void testMonitorStateAssertions() throws Exception { 145 ChannelID channelId1 = new ChannelID(1); 146 ThreadID txnId1 = new ThreadID(1); 147 148 ServerThreadContext thread1 = makeTxn(channelId1, txnId1); 149 150 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 151 lock.requestLock(thread1, LockLevel.WRITE, sink); 152 assertEquals(1, lock.getHoldersCount()); 153 assertEquals(0, lock.getWaiterCount()); 154 lock.wait(thread1, waitTimer, new WaitInvocation(), lockMgr, sink); assertEquals(0, lock.getHoldersCount()); 157 assertEquals(1, lock.getWaiterCount()); 158 159 try { 160 lock.wait(thread1, waitTimer, new WaitInvocation(), lockMgr, sink); 161 fail("able to join wait set twice"); 162 } catch (TCAssertionError e) { 163 } 165 166 try { 167 lock.notify(thread1, false, notifiedWaiters); 168 fail("able to call notify whilst being in the wait set"); 169 } catch (TCAssertionError e) { 170 } 172 } 173 174 public void testIllegalMonitorState() { 175 ChannelID goodChannelID = new ChannelID(1); 176 ThreadID goodTxnId = new ThreadID(1); 177 178 ChannelID badChannelID = new ChannelID(2); 179 ThreadID badTxnId = new ThreadID(2); 180 181 ServerThreadContext good = makeTxn(goodChannelID, goodTxnId); 182 ServerThreadContext bad = makeTxn(badChannelID, badTxnId); 183 184 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 185 lock.requestLock(good, LockLevel.WRITE, sink); 186 assertEquals(1, lock.getHoldersCount()); 187 assertFalse(lock.hasPending()); 188 189 try { 190 lock.wait(bad, waitTimer, new WaitInvocation(), lockMgr, sink); 192 fail("Not expected"); 193 } catch (TCIllegalMonitorStateException e) { 194 try { 195 lock.notify(bad, false, notifiedWaiters); 197 fail("Not expected"); 198 } catch (TCIllegalMonitorStateException e2) { 199 try { 200 lock.notify(bad, true, notifiedWaiters); 202 fail("Not expected"); 203 } catch (TCIllegalMonitorStateException e3) { 204 } 206 } 207 } 208 209 lock.removeCurrentHold(good); 211 lock.nextPending(); 212 assertEquals(0, lock.getHoldersCount()); 213 214 try { 215 lock.wait(good, waitTimer, new WaitInvocation(), lockMgr, sink); 217 fail("Not expected"); 218 } catch (TCIllegalMonitorStateException e) { 219 try { 220 lock.notify(good, false, notifiedWaiters); 222 fail("Not expected"); 223 } catch (TCIllegalMonitorStateException e2) { 224 try { 225 lock.notify(good, true, notifiedWaiters); 227 fail("Not expected"); 228 } catch (TCIllegalMonitorStateException e3) { 229 } 231 } 232 } 233 234 assertEquals(0, lock.getHoldersCount()); 236 lock.requestLock(good, LockLevel.READ, sink); 237 assertEquals(1, lock.getHoldersCount()); 238 239 try { 240 lock.wait(good, waitTimer, new WaitInvocation(), lockMgr, sink); 242 fail("Not expected"); 243 } catch (TCIllegalMonitorStateException e) { 244 try { 245 lock.notify(good, false, notifiedWaiters); 247 fail("Not expected"); 248 } catch (TCIllegalMonitorStateException e2) { 249 try { 250 lock.notify(good, true, notifiedWaiters); 252 fail("Not expected"); 253 } catch (TCIllegalMonitorStateException e3) { 254 } 256 } 257 } 258 } 259 260 public void testTimedWaitWithNotify() throws Exception { 261 ChannelID channelId1 = new ChannelID(1); 262 ThreadID txnId1 = new ThreadID(1); 263 ThreadID txnId2 = new ThreadID(2); 264 265 ServerThreadContext thread1 = makeTxn(channelId1, txnId1); 266 ServerThreadContext thread2 = makeTxn(channelId1, txnId2); 267 268 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 269 lock.requestLock(thread1, LockLevel.WRITE, sink); 270 271 assertEquals(1, lock.getHoldersCount()); 272 assertEquals(0, lock.getWaiterCount()); 273 assertFalse(lock.hasPending()); 274 275 lock.wait(thread1, waitTimer, new WaitInvocation(200), lockMgr, sink); 276 assertEquals(0, lock.getHoldersCount()); 277 assertEquals(1, lock.getWaiterCount()); 278 assertFalse(lock.hasPending()); 279 280 lock.requestLock(thread2, LockLevel.WRITE, sink); 281 lock.notify(thread2, false, notifiedWaiters); 282 assertEquals(1, lock.getHoldersCount()); 283 assertEquals(0, lock.getWaiterCount()); 284 assertEquals(1, lock.getPendingCount()); 285 286 ThreadUtil.reallySleep(2000); 288 assertEquals(1, lock.getHoldersCount()); 289 assertEquals(0, lock.getWaiterCount()); 290 assertEquals(1, lock.getPendingCount()); 291 } 292 293 public void testTimedWait2() throws Exception { 294 ChannelID channelId1 = new ChannelID(1); 296 ThreadID txnId1 = new ThreadID(1); 297 lockMgr.start(); 298 299 ServerThreadContext thread1 = makeTxn(channelId1, txnId1); 300 301 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 302 lock.requestLock(thread1, LockLevel.WRITE, sink); 303 assertEquals(1, lock.getHoldersCount()); 304 assertEquals(0, lock.getWaiterCount()); 305 assertFalse(lock.hasPending()); 306 307 long t1 = System.currentTimeMillis(); 308 lock.wait(thread1, waitTimer, new WaitInvocation(2000, 500), lockMgr, sink); 309 314 while(lock.getHoldersCount() != 1) { 315 ThreadUtil.reallySleep(100); 316 } 317 long t2 = System.currentTimeMillis(); 318 assertTrue(t2-t1 >= 2000); 319 320 assertEquals(1, lock.getHoldersCount()); 321 assertEquals(0, lock.getWaiterCount()); 322 assertFalse(lock.hasPending()); 323 } 324 325 public void testTimedWaitsDontFireWhenLockManagerIsStopped() throws Exception { 326 ChannelID channelId1 = new ChannelID(1); 327 ThreadID txnId1 = new ThreadID(1); 328 lockMgr.start(); 329 330 ServerThreadContext thread1 = makeTxn(channelId1, txnId1); 332 333 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 334 lock.requestLock(thread1, LockLevel.WRITE, sink); 335 assertEquals(1, lock.getHoldersCount()); 336 assertEquals(0, lock.getWaiterCount()); 337 assertFalse(lock.hasPending()); 338 339 lock.wait(thread1, waitTimer, new WaitInvocation(1000), lockMgr, sink); 340 assertEquals(0, lock.getHoldersCount()); 341 assertEquals(1, lock.getWaiterCount()); 342 assertFalse(lock.hasPending()); 343 344 ThreadUtil.reallySleep(250); 345 assertEquals(0, lock.getHoldersCount()); 346 assertEquals(1, lock.getWaiterCount()); 347 assertFalse(lock.hasPending()); 348 349 lockMgr.stop(); 351 352 ThreadUtil.reallySleep(250); 353 assertEquals(0, lock.getHoldersCount()); 354 assertEquals(1, lock.getWaiterCount()); 355 assertFalse(lock.hasPending()); 356 357 ThreadUtil.reallySleep(2000); 358 assertEquals(0, lock.getHoldersCount()); 359 assertEquals(1, lock.getWaiterCount()); 360 assertFalse(lock.hasPending()); 361 } 362 363 public void testTimedWaits() throws Exception { 364 ChannelID channelId1 = new ChannelID(1); 365 ThreadID txnId1 = new ThreadID(1); 366 ThreadID txnId2 = new ThreadID(2); 367 lockMgr.start(); 368 { 369 ServerThreadContext thread1 = makeTxn(channelId1, txnId1); 371 372 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 373 lock.requestLock(thread1, LockLevel.WRITE, sink); 374 assertEquals(1, lock.getHoldersCount()); 375 assertEquals(0, lock.getWaiterCount()); 376 assertFalse(lock.hasPending()); 377 378 long t1 = System.currentTimeMillis(); 379 lock.wait(thread1, waitTimer, new WaitInvocation(1000), lockMgr, sink); 380 385 while(lock.getHoldersCount() != 1) { 386 ThreadUtil.reallySleep(100); 387 } 388 long t2 = System.currentTimeMillis(); 389 assertTrue(t2-t1 >= 1000); 390 391 assertEquals(1, lock.getHoldersCount()); 392 assertEquals(0, lock.getWaiterCount()); 393 assertFalse(lock.hasPending()); 394 395 } 396 397 { 398 ServerThreadContext thread1 = makeTxn(channelId1, txnId1); 402 ServerThreadContext thread2 = makeTxn(channelId1, txnId2); 403 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 404 lock.requestLock(thread1, LockLevel.WRITE, sink); 405 assertEquals(1, lock.getHoldersCount()); 406 assertEquals(0, lock.getWaiterCount()); 407 assertFalse(lock.hasPending()); 408 409 lock.wait(thread1, waitTimer, new WaitInvocation(500), lockMgr, sink); 410 assertEquals(0, lock.getHoldersCount()); 411 assertEquals(1, lock.getWaiterCount()); 412 assertFalse(lock.hasPending()); 413 414 lock.requestLock(thread2, LockLevel.WRITE, sink); 415 assertEquals(1, lock.getHoldersCount()); 416 assertEquals(1, lock.getWaiterCount()); 417 assertFalse(lock.hasPending()); 418 419 ThreadUtil.reallySleep(1000); 420 assertEquals(1, lock.getHoldersCount()); 421 assertEquals(0, lock.getWaiterCount()); 422 assertEquals(1, lock.getPendingCount()); 423 424 lock.removeCurrentHold(thread2); 425 lock.nextPending(); 426 assertEquals(1, lock.getHoldersCount()); 427 assertEquals(0, lock.getWaiterCount()); 428 assertEquals(0, lock.getPendingCount()); 429 430 lock.removeCurrentHold(thread1); 431 lock.nextPending(); 432 assertEquals(0, lock.getHoldersCount()); 433 assertEquals(0, lock.getWaiterCount()); 434 assertEquals(0, lock.getPendingCount()); 435 } 436 } 437 438 public void testWait() throws Exception { 439 ChannelID channelID1 = new ChannelID(1); 440 ThreadID txnId1 = new ThreadID(1); 441 442 ServerThreadContext thread1 = makeTxn(channelID1, txnId1); 443 444 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 445 lock.requestLock(thread1, LockLevel.WRITE, sink); 446 assertEquals(1, lock.getHoldersCount()); 447 assertEquals(0, lock.getWaiterCount()); 448 assertFalse(lock.hasPending()); 449 450 lock.wait(thread1, waitTimer, new WaitInvocation(), lockMgr, sink); 451 452 assertEquals(0, lock.getHoldersCount()); 453 assertEquals(1, lock.getWaiterCount()); 454 assertFalse(lock.hasPending()); 455 } 456 457 public void testWaitOnUpgradedLock() throws Exception { 458 ChannelID channelID1 = new ChannelID(1); 459 460 ServerThreadContext thread1 = makeTxn(channelID1, new ThreadID(1)); 461 ServerThreadContext thread2 = makeTxn(channelID1, new ThreadID(2)); 462 463 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 464 lock.requestLock(thread1, LockLevel.READ, sink); 465 assertEquals(1, lock.getHoldersCount()); 466 assertEquals(0, lock.getWaiterCount()); 467 assertFalse(lock.hasPending()); 468 469 lock.requestLock(thread1, LockLevel.WRITE, sink); 470 assertEquals(1, lock.getHoldersCount()); 471 assertEquals(0, lock.getWaiterCount()); 472 assertFalse(lock.hasPending()); 473 474 lock.wait(thread1, waitTimer, new WaitInvocation(), lockMgr, sink); 475 assertEquals(0, lock.getHoldersCount()); 476 assertEquals(1, lock.getWaiterCount()); 477 assertFalse(lock.hasPending()); 478 479 boolean granted = lock.requestLock(thread2, LockLevel.WRITE, sink); 481 assertTrue(granted); 482 assertEquals(1, lock.getHoldersCount()); 483 assertEquals(1, lock.getWaiterCount()); 484 assertFalse(lock.hasPending()); 485 486 lock.notify(thread2, false, notifiedWaiters); 487 assertEquals(1, lock.getHoldersCount()); 488 assertEquals(0, lock.getWaiterCount()); 489 assertEquals(1, lock.getPendingCount()); 490 491 lock.removeCurrentHold(thread2); 492 lock.nextPending(); 493 assertEquals(1, lock.getHoldersCount()); 494 assertEquals(0, lock.getWaiterCount()); 495 assertEquals(0, lock.getPendingCount()); 496 497 Collection holders = lock.getHoldersCollection(); 499 assertEquals(1, holders.size()); 500 Holder holder = null; 501 for (Iterator iter = holders.iterator(); iter.hasNext();) { 502 assertNull(holder); holder = (Holder) iter.next(); 504 } 505 506 assertEquals(thread1, holder.getThreadContext()); 507 assertTrue(holder.isUpgrade()); 508 assertEquals(LockLevel.READ | LockLevel.WRITE, holder.getLockLevel()); 509 } 510 511 public void testNotifyAll() throws Exception { 512 Lock lock = createLockWithIndefiniteWaits(100); 513 514 assertEquals(0, lock.getHoldersCount()); 515 assertEquals(100, lock.getWaiterCount()); 516 assertEquals(0, lock.getPendingCount()); 517 518 ServerThreadContext notifier = makeTxn(getUniqueChannelID(), getUniqueTransactionID()); 519 lock.requestLock(notifier, LockLevel.WRITE, sink); 520 lock.notify(notifier, true, notifiedWaiters); 521 assertEquals(1, lock.getHoldersCount()); 522 assertEquals(0, lock.getWaiterCount()); 523 assertEquals(100, lock.getPendingCount()); 524 525 lock.removeCurrentHold(notifier); 526 lock.nextPending(); 527 assertEquals(1, lock.getHoldersCount()); 528 assertEquals(0, lock.getWaiterCount()); 529 assertEquals(99, lock.getPendingCount()); 530 } 531 532 public void testNotify() throws Exception { 533 Lock lock = createLockWithIndefiniteWaits(3); 534 535 assertEquals(0, lock.getHoldersCount()); 536 assertEquals(3, lock.getWaiterCount()); 537 assertEquals(0, lock.getPendingCount()); 538 539 ServerThreadContext notifier = makeTxn(getUniqueChannelID(), getUniqueTransactionID()); 540 lock.requestLock(notifier, LockLevel.WRITE, sink); 541 lock.notify(notifier, false, notifiedWaiters); 542 assertEquals(1, lock.getHoldersCount()); 543 assertEquals(2, lock.getWaiterCount()); 544 assertEquals(1, lock.getPendingCount()); 545 546 lock.notify(notifier, false, notifiedWaiters); 547 assertEquals(1, lock.getHoldersCount()); 548 assertEquals(1, lock.getWaiterCount()); 549 assertEquals(2, lock.getPendingCount()); 550 551 lock.notify(notifier, false, notifiedWaiters); 552 assertEquals(1, lock.getHoldersCount()); 553 assertEquals(0, lock.getWaiterCount()); 554 assertEquals(3, lock.getPendingCount()); 555 556 lock.notify(notifier, false, notifiedWaiters); 558 assertEquals(1, lock.getHoldersCount()); 559 assertEquals(0, lock.getWaiterCount()); 560 assertEquals(3, lock.getPendingCount()); 561 562 lock.removeCurrentHold(notifier); 563 lock.nextPending(); 564 assertEquals(1, lock.getHoldersCount()); 565 assertEquals(0, lock.getWaiterCount()); 566 assertEquals(2, lock.getPendingCount()); 567 } 568 569 private ChannelID getUniqueChannelID() { 570 return new ChannelID(uniqueId++); 571 } 572 573 private ThreadID getUniqueTransactionID() { 574 return new ThreadID(uniqueId++); 575 } 576 577 private Lock createLockWithIndefiniteWaits(int numWaits) throws TCIllegalMonitorStateException { 578 Lock lock = new Lock(new LockID("timmy"), 0, new LockEventListener[] {}); 579 580 for (int i = 0; i < numWaits; i++) { 581 int before = lock.getWaiterCount(); 582 583 ServerThreadContext me = makeTxn(getUniqueChannelID(), getUniqueTransactionID()); 584 lock.requestLock(me, LockLevel.WRITE, sink); 585 lock.wait(me, waitTimer, new WaitInvocation(), lockMgr, sink); 586 assertEquals(before + 1, lock.getWaiterCount()); 587 } 588 589 assertEquals(numWaits, lock.getWaiterCount()); 590 assertEquals(0, lock.getHoldersCount()); 591 assertEquals(0, lock.getPendingCount()); 592 593 return lock; 594 } 595 596 public void testAddPending() throws Exception { 597 LockEventMonitor monitor = new LockEventMonitor(); 598 Lock lock = new Lock(new LockID("yo"), 0, new LockEventListener[] { monitor }); 599 600 ServerThreadContext thread1 = makeTxn(getUniqueChannelID(), getUniqueTransactionID()); 601 ServerThreadContext thread2 = makeTxn(getUniqueChannelID(), getUniqueTransactionID()); 602 603 lock.requestLock(thread1, LockLevel.WRITE, sink); 604 lock.requestLock(thread2, LockLevel.WRITE, sink); 605 606 int pendingCount = 1; 612 613 Collection holders = lock.getHoldersCollection(); 614 assertEquals(1, holders.size()); 615 checkNotifyAddPendingCallContextForAllHolders(lock, monitor, pendingCount, holders); 616 } 617 618 private void checkNotifyAddPendingCallContextForAllHolders(Lock lock, LockEventMonitor monitor, int waiterCount, 619 Collection holders) throws Exception { 620 for (Iterator iter = holders.iterator(); iter.hasNext();) { 621 Holder holder = (Holder) iter.next(); 622 checkCallContext(monitor.waitForNotifyAddPending(0), lock.getLockID(), holder.getChannelID(), waiterCount); 623 } 624 } 625 626 private void checkCallContext(CallContext cc, LockID theLockId, ChannelID theChannelId, int waiterCount) { 627 assertNotNull(cc); 628 LockAwardContext ac = cc.ctxt; 629 assertEquals(theLockId, ac.getLockID()); 630 assertEquals(theChannelId, ac.getChannelID()); 631 assertEquals(waiterCount, cc.waiterCount); 632 } 633 634 } | Popular Tags |