Java > Open Source Codes > com > sleepycat > je > txn > LockManager


1 /*-
2  * See the file LICENSE for redistribution information.
3  *
4  * Copyright (c) 2002,2006 Oracle. All rights reserved.
5  *
6  * $Id: LockManager.java,v 1.118 2006/10/30 21:14:27 bostic Exp $
7  */
8
9 package com.sleepycat.je.txn;
10
11 import java.util.HashMap ;
12 import java.util.HashSet ;
13 import java.util.Iterator ;
14 import java.util.Map ;
15 import java.util.Set ;
16
17 import com.sleepycat.je.DatabaseException;
18 import com.sleepycat.je.DeadlockException;
19 import com.sleepycat.je.LockStats;
20 import com.sleepycat.je.RunRecoveryException;
21 import com.sleepycat.je.StatsConfig;
22 import com.sleepycat.je.config.EnvironmentParams;
23 import com.sleepycat.je.dbi.DatabaseImpl;
24 import com.sleepycat.je.dbi.DbConfigManager;
25 import com.sleepycat.je.dbi.EnvConfigObserver;
26 import com.sleepycat.je.dbi.EnvironmentImpl;
27 import com.sleepycat.je.dbi.MemoryBudget;
28 import com.sleepycat.je.dbi.RangeRestartException;
29 import com.sleepycat.je.latch.Latch;
30 import com.sleepycat.je.latch.LatchStats;
31 import com.sleepycat.je.latch.LatchSupport;
32
33 /**
34  * LockManager manages locks.
35  *
36  * Note that locks are counted as taking up part of the JE cache;
37  */
38 public abstract class LockManager implements EnvConfigObserver {
39
40     /*
41      * The total memory cost for a lock is the Lock object, plus its entry and
42      * key in the lock hash table.
43      *
44      * The addition and removal of Lock objects, and the corresponding cost of
45      * their hashmap entry and key are tracked through the LockManager.
46      */
47     static final long TOTAL_LOCK_OVERHEAD =
48         MemoryBudget.LOCK_OVERHEAD +
49         MemoryBudget.HASHMAP_ENTRY_OVERHEAD +
50         MemoryBudget.LONG_OVERHEAD;
51
52     private static final long REMOVE_TOTAL_LOCK_OVERHEAD =
53         0 - TOTAL_LOCK_OVERHEAD;
54
55     protected int nLockTables = 1;
56     protected Latch[] lockTableLatches;
57     private Map [] lockTables; // keyed by nodeId
58 private EnvironmentImpl envImpl;
59     private MemoryBudget memoryBudget;
60     //private Level traceLevel;
61
62     private long nRequests; // stats: number of time a request was made
63 private long nWaits; // stats: number of time a request blocked
64
65     private static RangeRestartException rangeRestartException =
66     new RangeRestartException();
67     private static boolean lockTableDump = false;
68
69     public LockManager(EnvironmentImpl envImpl)
70         throws DatabaseException {
71             
72     DbConfigManager configMgr = envImpl.getConfigManager();
73     nLockTables = configMgr.getInt(EnvironmentParams.N_LOCK_TABLES);
74     lockTables = new Map [nLockTables];
75     lockTableLatches = new Latch[nLockTables];
76     for (int i = 0; i < nLockTables; i++) {
77         lockTables[i] = new HashMap ();
78         lockTableLatches[i] =
79         LatchSupport.makeLatch("Lock Table " + i, envImpl);
80     }
81         this.envImpl = envImpl;
82         memoryBudget = envImpl.getMemoryBudget();
83         nRequests = 0;
84         nWaits = 0;
85     /*
86         traceLevel = Tracer.parseLevel
87         (env, EnvironmentParams.JE_LOGGING_LEVEL_LOCKMGR);
88     */
89
90         /* Initialize mutable properties and register for notifications. */
91         envConfigUpdate(configMgr);
92         envImpl.addConfigObserver(this);
93     }
94
95     /**
96      * Process notifications of mutable property changes.
97      */
98     public void envConfigUpdate(DbConfigManager configMgr)
99         throws DatabaseException {
100
101         LockInfo.setDeadlockStackTrace(configMgr.getBoolean
102             (EnvironmentParams.TXN_DEADLOCK_STACK_TRACE));
103         setLockTableDump(configMgr.getBoolean
104             (EnvironmentParams.TXN_DUMPLOCKS));
105     }
106
107     /**
108      * Called when the je.txn.dumpLocks property is changed.
109      */
110     static void setLockTableDump(boolean enable) {
111         lockTableDump = enable;
112     }
113
114     protected int getLockTableIndex(Long nodeId) {
115     return ((int) nodeId.longValue()) %
116         nLockTables;
117     }
118
119     protected int getLockTableIndex(long nodeId) {
120     return ((int) nodeId) % nLockTables;
121     }
122
123     /**
124      * Attempt to acquire a lock of <i>type</i> on <i>nodeId</i>. If the lock
125      * acquisition would result in a deadlock, throw an exception.<br> If the
126      * requested lock is not currently available, block until it is or until
127      * timeout milliseconds have elapsed.<br> If a lock of <i>type</i> is
128      * already held, return EXISTING.<br> If a WRITE lock is held and a READ
129      * lock is requested, return PROMOTION.<br>
130      *
131      * If a lock request is for a lock that is not currently held, return
132      * either NEW or DENIED depending on whether the lock is granted or
133      * not.<br>
134      *
135      * @param nodeId The NodeId to lock.
136      *
137      * @param locker The Locker to lock this on behalf of.
138      *
139      * @param type The lock type requested.
140      *
141      * @param timeout milliseconds to time out after if lock couldn't be
142      * obtained. 0 means block indefinitely. Not used if nonBlockingRequest
143      * is true.
144      *
145      * @param nonBlockingRequest if true, means don't block if lock can't be
146      * acquired, and ignore the timeout parameter.
147      *
148      * @return a LockGrantType indicating whether the request was fulfilled
149      * or not. LockGrantType.NEW means the lock grant was fulfilled and
150      * the caller did not previously hold the lock. PROMOTION means the
151      * lock was granted and it was a promotion from READ to WRITE. EXISTING
152      * means the lock was already granted (not a promotion). DENIED means
153      * the lock was not granted either because the timeout passed without
154      * acquiring the lock or timeout was -1 and the lock was not immediately
155      * available.
156      *
157      * @throws DeadlockException if acquiring the lock would result in
158      * a deadlock.
159      */
160     public LockGrantType lock(long nodeId,
161                               Locker locker,
162                               LockType type,
163                               long timeout,
164                               boolean nonBlockingRequest,
165                   DatabaseImpl database)
166         throws DeadlockException, DatabaseException {
167
168         assert timeout >= 0;
169
170         /*
171          * Lock on locker before latching the lockTable to avoid having another
172          * notifier perform the notify before the waiter is actually waiting.
173          */
174         synchronized (locker) {
175             Long nid = new Long (nodeId);
176             LockAttemptResult result =
177         attemptLock(nid, locker, type, nonBlockingRequest);
178             /* Got the lock, return. */
179             if (result.success ||
180                 result.lockGrant == LockGrantType.DENIED) {
181                 return result.lockGrant;
182             }
183
184             assert checkNoLatchesHeld(nonBlockingRequest):
185                 LatchSupport.countLatchesHeld() +
186                 " latches held while trying to lock, lock table =" +
187                 LatchSupport.latchesHeldToString();
188
189             /*
190              * We must have gotten WAIT_* from the lock request. We know that
191              * this is a blocking request, because if it wasn't, Lock.lock
192              * would have returned DENIED. Go wait!
193              */
194             assert !nonBlockingRequest;
195             try {
196                 boolean doWait = true;
197
198                 /*
199                  * Before blocking, check locker timeout. We need to check here
200                  * or lock timeouts will always take precedence and we'll never
201                  * actually get any txn timeouts.
202                  */
203                 if (locker.isTimedOut()) {
204                     if (validateOwnership(nid, locker, type, true,
205                       memoryBudget)) {
206                         doWait = false;
207                     } else {
208                         String errMsg =
209                 makeTimeoutMsg("Transaction", locker, nodeId, type,
210                        result.lockGrant,
211                        result.useLock,
212                        locker.getTxnTimeOut(),
213                        locker.getTxnStartMillis(),
214                        System.currentTimeMillis(),
215                        database);
216                         throw new DeadlockException(errMsg);
217                     }
218                 }
219
220                 boolean keepTime = (timeout > 0);
221                 long startTime = (keepTime ? System.currentTimeMillis() : 0);
222                 while (doWait) {
223                     locker.setWaitingFor(result.useLock);
224                     try {
225                         locker.wait(timeout);
226                     } catch (InterruptedException IE) {
227             throw new RunRecoveryException(envImpl, IE);
228                     }
229
230                     boolean lockerTimedOut = locker.isTimedOut();
231                     long now = System.currentTimeMillis();
232                     boolean thisLockTimedOut =
233                         (keepTime && (now - startTime > timeout));
234                     boolean isRestart =
235                         (result.lockGrant == LockGrantType.WAIT_RESTART);
236
237                     /*
238                      * Re-check for ownership of the lock following wait. If
239                      * we timed out and we don't have ownership then flush this
240                      * lock from both the waiters and owners while under the
241                      * lock table latch. See SR 10103.
242                      */
243                     if (validateOwnership(nid, locker, type,
244                       lockerTimedOut ||
245                                           thisLockTimedOut ||
246                                           isRestart,
247                                           memoryBudget)) {
248                         break;
249                     } else {
250
251                         /*
252                          * After a restart conflict the lock will not be held.
253                          */
254                         if (isRestart) {
255                             throw rangeRestartException;
256                         }
257
258                         if (thisLockTimedOut) {
259                 locker.setOnlyAbortable();
260                             String errMsg =
261                                 makeTimeoutMsg("Lock", locker, nodeId, type,
262                                                result.lockGrant,
263                                                result.useLock,
264                                                timeout, startTime, now,
265                            database);
266                             throw new DeadlockException(errMsg);
267                         }
268
269                         if (lockerTimedOut) {
270                 locker.setOnlyAbortable();
271                             String errMsg =
272                                 makeTimeoutMsg("Transaction", locker,
273                                                nodeId, type,
274                                                result.lockGrant,
275                                                result.useLock,
276                                                locker.getTxnTimeOut(),
277                                                locker.getTxnStartMillis(),
278                                                now, database);
279                             throw new DeadlockException(errMsg);
280                         }
281                     }
282                 }
283             } finally {
284         locker.setWaitingFor(null);
285         assert EnvironmentImpl.maybeForceYield();
286         }
287
288             locker.addLock(nid, result.useLock, type, result.lockGrant);
289
290             return result.lockGrant;
291         }
292     }
293
294     abstract protected LockAttemptResult
295         attemptLock(Long nodeId,
296                     Locker locker,
297                     LockType type,
298                     boolean nonBlockingRequest)
299         throws DatabaseException;
300         
301     protected LockAttemptResult
302     attemptLockInternal(Long nodeId,
303                 Locker locker,
304                 LockType type,
305                 boolean nonBlockingRequest,
306                 int lockTableIndex)
307         throws DatabaseException {
308
309         nRequests++;
310
311         /* Get the target lock. */
312     Map lockTable = lockTables[lockTableIndex];
313         Lock useLock = (Lock) lockTable.get(nodeId);
314         if (useLock == null) {
315             useLock = new Lock(nodeId);
316             lockTable.put(nodeId, useLock);
317             memoryBudget.updateLockMemoryUsage(TOTAL_LOCK_OVERHEAD,
318                            lockTableIndex);
319         }
320
321         /*
322          * Attempt to lock. Possible return values are NEW, PROMOTION, DENIED,
323          * EXISTING, WAIT_NEW, WAIT_PROMOTION, WAIT_RESTART.
324          */
325         LockGrantType lockGrant = useLock.lock(type, locker,
326                            nonBlockingRequest,
327                            memoryBudget, lockTableIndex);
328         boolean success = false;
329
330         /* Was the attempt successful? */
331         if ((lockGrant == LockGrantType.NEW) ||
332             (lockGrant == LockGrantType.PROMOTION)) {
333             locker.addLock(nodeId, useLock, type, lockGrant);
334             success = true;
335         } else if (lockGrant == LockGrantType.EXISTING) {
336             success = true;
337         } else if (lockGrant == LockGrantType.DENIED) {
338             /* Locker.lock will throw LockNotGrantedException. */
339         } else {
340             nWaits++;
341         }
342         return new LockAttemptResult(useLock, lockGrant, success);
343     }
344
345     /**
346      * Create a informative lock or txn timeout message.
347      */
348     protected abstract String makeTimeoutMsg(String lockOrTxn,
349                                              Locker locker,
350                                              long nodeId,
351                                              LockType type,
352                                              LockGrantType grantType,
353                                              Lock useLock,
354                                              long timeout,
355                                              long start,
356                                              long now,
357                          DatabaseImpl database)
358     throws DatabaseException;
359
360     /**
361      * Do the real work of creating an lock or txn timeout message.
362      */
363     protected String makeTimeoutMsgInternal(String lockOrTxn,
364                                             Locker locker,
365                                             long nodeId,
366                                             LockType type,
367                                             LockGrantType grantType,
368                                             Lock useLock,
369                                             long timeout,
370                                             long start,
371                                             long now,
372                         DatabaseImpl database) {
373
374         /*
375          * Because we're accessing parts of the lock, need to have protected
376          * access to the lock table because things can be changing out from
377          * underneath us. This is a big hammer to grab for so long while we
378          * traverse the graph, but it's only when we have a deadlock and we're
379          * creating a debugging message.
380          *
381          * The alternative would be to handle ConcurrentModificationExceptions
382          * and retry until none of them happen.
383          */
384         if (lockTableDump) {
385             System.out.println("++++++++++ begin lock table dump ++++++++++");
386             for (int i = 0; i < nLockTables; i++) {
387                 StringBuffer sb = new StringBuffer ();
388                 dumpToStringNoLatch(sb, i);
389                 System.out.println(sb.toString());
390             }
391             System.out.println("++++++++++ end lock table dump ++++++++++");
392         }
393
394         StringBuffer sb = new StringBuffer ();
395         sb.append(lockOrTxn);
396         sb.append(" expired. Locker ").append(locker);
397         sb.append(": waited for lock");
398
399         if (database!=null) {
400             sb.append(" on database=").append(database.getDebugName());
401         }
402         sb.append(" node=").append(nodeId);
403         sb.append(" type=").append(type);
404         sb.append(" grant=").append(grantType);
405         sb.append(" timeoutMillis=").append(timeout);
406         sb.append(" startTime=").append(start);
407         sb.append(" endTime=").append(now);
408         sb.append("\nOwners: ").append(useLock.getOwnersClone());
409         sb.append("\nWaiters: ").append(useLock.getWaitersListClone()).
410         append("\n");
411         StringBuffer deadlockInfo = findDeadlock(useLock, locker);
412         if (deadlockInfo != null) {
413             sb.append(deadlockInfo);
414         }
415         return sb.toString();
416     }
417
418     /**
419      * Release a lock and possibly notify any waiters that they have been
420      * granted the lock.
421      *
422      * @param nodeId The node ID of the lock to release.
423      *
424      * @return true if the lock is released successfully, false if
425      * the lock is not currently being held.
426      */
427     boolean release(long nodeId, Locker locker)
428         throws DatabaseException {
429
430         return release(nodeId, null, locker, true);
431     }
432
433     /**
434      * Release a lock and possibly notify any waiters that they have been
435      * granted the lock.
436      *
437      * @param lock The lock to release
438      *
439      * @return true if the lock is released successfully, false if the lock is
440      * not currently being held.
441      */
442     boolean release(Lock lock, Locker locker)
443         throws DatabaseException {
444
445         return release(-1, lock, locker, false);
446     }
447
448     /**
449      * Do the work of releasing a lock and notifying any waiters that they have
450      * been granted the lock.
451      *
452      * @param lock The lock to release. If null, use nodeId to find lock
453      * @param nodeId The node ID of the lock to release, if lock is null. May
454      * not be valid if lock is not null. MUST be valid if removeFromLocker is
455      * true
456      * @param locker
457      * @param removeFromLocker true if we're responsible for
458      *
459      * @return true if the lock is released successfully, false if the lock is
460      * not currently being held.
461      */
462     private boolean release(long nodeId,
463                             Lock lock,
464                             Locker locker,
465                             boolean removeFromLocker)
466         throws DatabaseException {
467
468     synchronized (locker) {
469         Set newOwners =
470         releaseAndFindNotifyTargets(nodeId, lock, locker,
471                         removeFromLocker);
472
473             if (newOwners == null) {
474                 return false;
475             }
476
477             if (newOwners.size() > 0) {
478
479                 /*
480                  * There is a new set of owners and/or there are restart
481                  * waiters that should be notified.
482                  */
483                 Iterator iter = newOwners.iterator();
484                 
485                 while (iter.hasNext()) {
486                     Locker lockerToNotify = (Locker) iter.next();
487
488                     /* Use notifyAll to support multiple threads per txn. */
489                     synchronized (lockerToNotify) {
490                         lockerToNotify.notifyAll();
491                     }
492
493             assert EnvironmentImpl.maybeForceYield();
494                 }
495             }
496
497             return true;
498     }
499     }
500
501     /**
502      * Release the lock, and return the set of new owners to notify, if any.
503      *
504      * @return
505      * null if the lock does not exist or the given locker was not the owner,
506      * a non-empty set if owners should be notified after releasing,
507      * an empty set if no notification is required.
508      */
509     protected abstract Set
510         releaseAndFindNotifyTargets(long nodeId,
511                                     Lock lock,
512                                     Locker locker,
513                                     boolean removeFromLocker)
514         throws DatabaseException;
515
516     /**
517      * Do the real work of releaseAndFindNotifyTargets
518      */
519     protected Set
520     releaseAndFindNotifyTargetsInternal(long nodeId,
521                         Lock lock,
522                         Locker locker,
523                         boolean removeFromLocker,
524                         int lockTableIndex)
525         throws DatabaseException {
526
527         Lock useLock = lock;
528
529     Map lockTable = lockTables[lockTableIndex];
530         if (useLock == null) {
531         useLock = (Lock) lockTable.get(new Long (nodeId));
532         }
533                 
534         if (useLock == null) {
535             /* Lock doesn't exist. */
536             return null;
537         }
538
539         Set lockersToNotify =
540         useLock.release(locker, memoryBudget, lockTableIndex);
541         if (lockersToNotify == null) {
542             /* Not owner. */
543             return null;
544         }
545
546         /*
547          * If desired, remove it from the locker's bag. Used when we don't need
548          * to hang onto the lock after release -- like null txns or locks on
549          * deleted LNs.
550          */
551         if (removeFromLocker) {
552             assert nodeId != -1;
553             locker.removeLock(nodeId, useLock);
554         }
555
556         /* If it's not in use at all, remove it from the lock table. */
557         if ((useLock.nWaiters() == 0) &&
558             (useLock.nOwners() == 0)) {
559             lockTables[lockTableIndex].remove(useLock.getNodeId());
560             memoryBudget.updateLockMemoryUsage(REMOVE_TOTAL_LOCK_OVERHEAD,
561                            lockTableIndex);
562         }
563
564         return lockersToNotify;
565     }
566
567     /**
568      * Transfer ownership a lock from one locker to another locker. We're not
569      * sending any notification to the waiters on the lock table, and the past
570      * and present owner should be ready for the transfer.
571      */
572     abstract void transfer(long nodeId,
573                            Locker owningLocker,
574                            Locker destLocker,
575                            boolean demoteToRead)
576         throws DatabaseException;
577
578     /**
579      * Do the real work of transfer
580      */
581     protected void transferInternal(long nodeId,
582                                     Locker owningLocker,
583                                     Locker destLocker,
584                                     boolean demoteToRead,
585                     int lockTableIndex)
586         throws DatabaseException {
587         
588     Map lockTable = lockTables[lockTableIndex];
589         Lock useLock = (Lock) lockTable.get(new Long (nodeId));
590         
591         assert useLock != null : "Transfer, lock " + nodeId + " was null";
592         if (demoteToRead) {
593             useLock.demote(owningLocker);
594         }
595     useLock.transfer(owningLocker, destLocker,
596              memoryBudget, lockTableIndex);
597         owningLocker.removeLock(nodeId, useLock);
598     }
599     
600     /**
601      * Transfer ownership a lock from one locker to a set of other txns,
602      * cloning the lock as necessary. This will always be demoted to read, as
603      * we can't have multiple locker owners any other way. We're not sending
604      * any notification to the waiters on the lock table, and the past and
605      * present owners should be ready for the transfer.
606      */
607     abstract void transferMultiple(long nodeId,
608                                    Locker owningLocker,
609                                    Locker[] destLockers)
610         throws DatabaseException;
611
612     /**
613      * Do the real work of transferMultiple
614      */
615     protected void transferMultipleInternal(long nodeId,
616                                             Locker owningLocker,
617                                             Locker[] destLockers,
618                         int lockTableIndex)
619         throws DatabaseException {
620
621     Map lockTable = lockTables[lockTableIndex];
622         Lock useLock = (Lock) lockTable.get(new Long (nodeId));
623             
624         assert useLock != null : "Transfer, lock " + nodeId + " was null";
625         useLock.demote(owningLocker);
626         useLock.transferMultiple(owningLocker, destLockers,
627                  memoryBudget, lockTableIndex);
628         owningLocker.removeLock(nodeId, useLock);
629     }
630
631     /**
632      * Demote a lock from write to read. Call back to the owning locker to
633      * move this to its read collection.
634      * @param lock The lock to release. If null, use nodeId to find lock
635      * @param locker
636      */
637     abstract void demote(long nodeId, Locker locker)
638         throws DatabaseException;
639
640     /**
641      * Do the real work of demote.
642      */
643     protected void demoteInternal(long nodeId,
644                   Locker locker,
645                   int lockTableIndex)
646         throws DatabaseException {
647
648     Map lockTable = lockTables[lockTableIndex];
649         Lock useLock = (Lock) lockTable.get(new Long (nodeId));
650         useLock.demote(locker);
651         locker.moveWriteToReadLock(nodeId, useLock);
652     }
653
654     /**
655      * Test the status of the lock on nodeId. If any transaction holds any
656      * lock on it, true is returned. If no transaction holds a lock on it,
657      * false is returned.
658      *
659      * This method is only used by unit tests.
660      *
661      * @param nodeId The NodeId to check.
662      * @return true if any transaction holds any lock on the nodeid. false
663      * if no lock is held by any transaction.
664      */
665     abstract boolean isLocked(Long nodeId)
666         throws DatabaseException;
667
668     /**
669      * Do the real work of isLocked.
670      */
671     protected boolean isLockedInternal(Long nodeId, int lockTableIndex) {
672
673     Map lockTable = lockTables[lockTableIndex];
674         Lock entry = (Lock) lockTable.get(nodeId);
675         if (entry == null) {
676             return false;
677         }
678
679         return entry.nOwners() != 0;
680     }
681     
682     /**
683      * Return true if this locker owns this a lock of this type on given node.
684      *
685      * This method is only used by unit tests.
686      */
687     abstract boolean isOwner(Long nodeId, Locker locker, LockType type)
688         throws DatabaseException;
689
690     /**
691      * Do the real work of isOwner.
692      */
693     protected boolean isOwnerInternal(Long nodeId,
694                                       Locker locker,
695                                       LockType type,
696                       int lockTableIndex) {
697
698     Map lockTable = lockTables[lockTableIndex];
699         Lock entry = (Lock) lockTable.get(nodeId);
700         if (entry == null) {
701             return false;
702         }
703
704         return entry.isOwner(locker, type);
705     }
706
707     /**
708      * Return true if this locker is waiting on this lock.
709      *
710      * This method is only used by unit tests.
711      */
712     abstract boolean isWaiter(Long nodeId, Locker locker)
713         throws DatabaseException;
714
715     /**
716      * Do the real work of isWaiter.
717      */
718     protected boolean isWaiterInternal(Long nodeId,
719                        Locker locker,
720                        int lockTableIndex) {
721
722     Map lockTable = lockTables[lockTableIndex];
723         Lock entry = (Lock) lockTable.get(nodeId);
724         if (entry == null) {
725             return false;
726         }
727
728         return entry.isWaiter(locker);
729     }
730
731     /**
732      * Return the number of waiters for this lock.
733      */
734     abstract int nWaiters(Long nodeId)
735         throws DatabaseException;
736
737     /**
738      * Do the real work of nWaiters.
739      */
740     protected int nWaitersInternal(Long nodeId, int lockTableIndex) {
741
742     Map lockTable = lockTables[lockTableIndex];
743         Lock entry = (Lock) lockTable.get(nodeId);
744         if (entry == null) {
745             return -1;
746         }
747
748         return entry.nWaiters();
749     }
750
751     /**
752      * Return the number of owners of this lock.
753      */
754     abstract int nOwners(Long nodeId)
755         throws DatabaseException;
756
757     /**
758      * Do the real work of nWaiters.
759      */
760     protected int nOwnersInternal(Long nodeId, int lockTableIndex) {
761
762     Map lockTable = lockTables[lockTableIndex];
763         Lock entry = (