KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > jboss > ha > hasessionstate > server > HASessionStateImpl


1 /*
2   * JBoss, Home of Professional Open Source
3   * Copyright 2005, JBoss Inc., and individual contributors as indicated
4   * by the @authors tag. See the copyright.txt in the distribution for a
5   * full listing of individual contributors.
6   *
7   * This is free software; you can redistribute it and/or modify it
8   * under the terms of the GNU Lesser General Public License as
9   * published by the Free Software Foundation; either version 2.1 of
10   * the License, or (at your option) any later version.
11   *
12   * This software is distributed in the hope that it will be useful,
13   * but WITHOUT ANY WARRANTY; without even the implied warranty of
14   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15   * Lesser General Public License for more details.
16   *
17   * You should have received a copy of the GNU Lesser General Public
18   * License along with this software; if not, write to the Free
19   * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
20   * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
21   */

22 package org.jboss.ha.hasessionstate.server;
23
24 import java.io.ByteArrayInputStream JavaDoc;
25 import java.io.ByteArrayOutputStream JavaDoc;
26 import java.io.IOException JavaDoc;
27 import java.io.ObjectInputStream JavaDoc;
28 import java.io.ObjectOutputStream JavaDoc;
29 import java.io.Serializable JavaDoc;
30 import java.util.ArrayList JavaDoc;
31 import java.util.Enumeration JavaDoc;
32 import java.util.HashMap JavaDoc;
33 import java.util.Hashtable JavaDoc;
34 import java.util.Iterator JavaDoc;
35 import java.util.Vector JavaDoc;
36 import java.util.zip.Deflater JavaDoc;
37 import java.util.zip.DeflaterOutputStream JavaDoc;
38 import java.util.zip.InflaterInputStream JavaDoc;
39
40 import javax.naming.Context JavaDoc;
41 import javax.naming.InitialContext JavaDoc;
42 import javax.naming.Name JavaDoc;
43 import javax.naming.NameNotFoundException JavaDoc;
44 import javax.naming.Reference JavaDoc;
45 import javax.naming.StringRefAddr JavaDoc;
46
47 import org.jboss.ha.framework.interfaces.HAPartition;
48 import org.jboss.ha.hasessionstate.interfaces.PackagedSession;
49 import org.jboss.logging.Logger;
50 import org.jboss.naming.NonSerializableFactory;
51 import org.jboss.system.server.ServerConfigUtil;
52
53 import EDU.oswego.cs.dl.util.concurrent.Mutex;
54
55 /**
56  * Default implementation of HASessionState
57  *
58  * @see org.jboss.ha.hasessionstate.interfaces.HASessionState
59  * @author sacha.labourey@cogito-info.ch
60  * @author <a HREF="bill@burkecentral.com">Bill Burke</a>
61  * @version $Revision: 43857 $
62  *
63  * <p><b>Revisions:</b><br>
64  * <p><b>2002/01/09: billb</b>
65  * <ol>
66  * <li>ripped out sub partitioning stuff. It really belongs as a subclass of HAPartition
67  * </ol>
68  *
69  */

70
71 public class HASessionStateImpl
72    implements org.jboss.ha.hasessionstate.interfaces.HASessionState,
73           HAPartition.HAPartitionStateTransfer
74 {
75    
76    protected String JavaDoc _sessionStateName;
77    protected Logger log;
78    protected HAPartition hapGeneral;
79    protected String JavaDoc sessionStateIdentifier;
80    protected String JavaDoc myNodeName;
81    
82    protected long beanCleaningDelay;
83    protected String JavaDoc haPartitionName;
84    protected String JavaDoc haPartitionJndiName;
85    
86    protected final String JavaDoc DEFAULT_PARTITION_JNDI_NAME = ServerConfigUtil.getDefaultPartitionName();
87    protected final String JavaDoc JNDI_FOLDER_NAME_FOR_HASESSIONSTATE = org.jboss.metadata.ClusterConfigMetaData.JNDI_PREFIX_FOR_SESSION_STATE;
88    protected final String JavaDoc JNDI_FOLDER_NAME_FOR_HAPARTITION = "/HAPartition/";
89    protected final long MAX_DELAY_BEFORE_CLEANING_UNRECLAIMED_STATE = 30L * 60L * 1000L; // 30 minutes... should be set externally or use cache settings
90
protected static final String JavaDoc HA_SESSION_STATE_STATE_TRANSFER = "HASessionStateTransfer";
91    
92    protected HashMap JavaDoc locks = new HashMap JavaDoc ();
93       
94    public HASessionStateImpl ()
95    {}
96    
97    public HASessionStateImpl (String JavaDoc sessionStateName,
98                               HAPartition partition,
99                               long beanCleaningDelay)
100    {
101       this(sessionStateName, partition.getPartitionName(), beanCleaningDelay);
102       this.hapGeneral = partition;
103    }
104    
105    public HASessionStateImpl (String JavaDoc sessionStateName,
106                               String JavaDoc mainHAPartitionName,
107                               long beanCleaningDelay)
108    {
109       if (sessionStateName == null)
110          this._sessionStateName = org.jboss.metadata.ClusterConfigMetaData.DEFAULT_SESSION_STATE_NAME;
111       else
112          this._sessionStateName = sessionStateName;
113       
114       this.sessionStateIdentifier = "SessionState-'" + this._sessionStateName + "'";
115       
116       if (mainHAPartitionName == null)
117          haPartitionName = DEFAULT_PARTITION_JNDI_NAME;
118       else
119          haPartitionName = mainHAPartitionName;
120       
121       haPartitionJndiName = JNDI_FOLDER_NAME_FOR_HAPARTITION + haPartitionName;
122       
123       if (beanCleaningDelay > 0)
124          this.beanCleaningDelay = beanCleaningDelay;
125       else
126          this.beanCleaningDelay = MAX_DELAY_BEFORE_CLEANING_UNRECLAIMED_STATE;
127       
128    }
129    
130    public void init () throws Exception JavaDoc
131    {
132       this.log = Logger.getLogger(HASessionStateImpl.class.getName() + "." + this._sessionStateName);
133
134       // BES 20060416 -- if people used an old config, we may not
135
// have been passed the partition, so have to find in JNDI
136
// JNDI work s/b done in start(), but we have no choice, as
137
// we must register for state transfer in init
138
if (this.hapGeneral == null)
139       {
140          Context JavaDoc ctx = new InitialContext JavaDoc ();
141          this.hapGeneral = (HAPartition)ctx.lookup (haPartitionJndiName);
142       }
143       
144       if (hapGeneral == null)
145          log.error ("Unable to get default HAPartition under name '" + haPartitionJndiName + "'.");
146       
147       this.hapGeneral.registerRPCHandler (this.sessionStateIdentifier, this);
148       this.hapGeneral.subscribeToStateTransferEvents (HA_SESSION_STATE_STATE_TRANSFER, this);
149    }
150    
151    public void start () throws Exception JavaDoc
152    {
153       this.myNodeName = this.hapGeneral.getNodeName ();
154       log.debug ("HASessionState node name : " + this.myNodeName );
155       
156       // BES 4/7/06 clean up lifecycle; move this to start, as it can't be
157
// called until startService due to JNDI dependency
158
Context JavaDoc ctx = new InitialContext JavaDoc ();
159       this.bind (this._sessionStateName, this, HASessionStateImpl.class, ctx);
160    }
161    
162    protected void bind (String JavaDoc jndiName, Object JavaDoc who, Class JavaDoc classType, Context JavaDoc ctx) throws Exception JavaDoc
163    {
164       // Ah ! This service isn't serializable, so we use a helper class
165
//
166
NonSerializableFactory.bind (jndiName, who);
167       Name JavaDoc n = ctx.getNameParser ("").parse (jndiName);
168       while (n.size () > 1)
169       {
170          String JavaDoc ctxName = n.get (0);
171          try
172          {
173             ctx = (Context JavaDoc)ctx.lookup (ctxName);
174          }
175          catch (NameNotFoundException JavaDoc e)
176          {
177             log.debug ("creating Subcontext" + ctxName);
178             ctx = ctx.createSubcontext (ctxName);
179          }
180          n = n.getSuffix (1);
181       }
182       
183       // The helper class NonSerializableFactory uses address type nns, we go on to
184
// use the helper class to bind the service object in JNDI
185
//
186
StringRefAddr JavaDoc addr = new StringRefAddr JavaDoc ("nns", jndiName);
187       Reference JavaDoc ref = new Reference JavaDoc ( classType.getName (), addr, NonSerializableFactory.class.getName (), null);
188       ctx.bind (n.get (0), ref);
189    }
190    
191    public void stop () throws Exception JavaDoc
192    {
193        purgeState();
194        
195        // Unbind so we can rebind if restarted
196
try
197        {
198           Context JavaDoc ctx = new InitialContext JavaDoc ();
199           ctx.unbind (this._sessionStateName);
200           NonSerializableFactory.unbind (this._sessionStateName);
201        }
202        catch (Exception JavaDoc ignored)
203        {}
204    }
205    
206    public void destroy() throws Exception JavaDoc
207    {
208       // Remove ref to ourself from HAPartition
209
this.hapGeneral.unregisterRPCHandler(this.sessionStateIdentifier, this);
210       this.hapGeneral.unsubscribeFromStateTransferEvents(HA_SESSION_STATE_STATE_TRANSFER, this);
211    }
212    
213    public String JavaDoc getNodeName ()
214    {
215       return this.myNodeName ;
216    }
217    
218    // Used for Session state transfer
219
//
220
public Serializable JavaDoc getCurrentState ()
221    {
222       log.debug ("Building and returning state of HASessionState");
223       
224       if (this.appSessions == null)
225          this.appSessions = new Hashtable JavaDoc ();
226       
227       Serializable JavaDoc result = null;
228       
229       synchronized (this.lockAppSession)
230       {
231          this.purgeState ();
232          
233          try
234          {
235             result = deflate (this.appSessions);
236          }
237          catch (Exception JavaDoc e)
238          {
239             log.error("operation failed", e);
240          }
241       }
242       return result;
243    }
244    
245    public void setCurrentState (Serializable JavaDoc newState)
246    {
247       log.debug ("Receiving state of HASessionState");
248       
249       if (this.appSessions == null)
250          this.appSessions = new Hashtable JavaDoc ();
251       
252       synchronized (this.lockAppSession)
253       {
254          try
255          {
256             this.appSessions.clear (); // hope to facilitate the job of the GC
257
this.appSessions = (Hashtable JavaDoc)inflate ((byte[])newState);
258          }
259          catch (Exception JavaDoc e)
260          {
261             log.error("operation failed", e);
262          }
263       }
264    }
265    
266    public void purgeState ()
267    {
268       synchronized (this.lockAppSession)
269       {
270          for (Enumeration JavaDoc keyEnum = this.appSessions.keys (); keyEnum.hasMoreElements ();)
271          {
272             // trip in apps..
273
//
274
Object JavaDoc key = keyEnum.nextElement ();
275             Hashtable JavaDoc value = (Hashtable JavaDoc)this.appSessions.get (key);
276             long currentTime = System.currentTimeMillis ();
277             
278             for (Iterator JavaDoc iterSessions = value.values ().iterator (); iterSessions.hasNext ();)
279             {
280                PackagedSession ps = (PackagedSession)iterSessions.next ();
281                if ( (currentTime - ps.unmodifiedExistenceInVM ()) > beanCleaningDelay )
282                   iterSessions.remove ();
283             }
284          }
285       }
286       
287    }
288    
289    protected byte[] deflate (Object JavaDoc object) throws IOException JavaDoc
290    {
291       ByteArrayOutputStream JavaDoc baos = new ByteArrayOutputStream JavaDoc ();
292       Deflater JavaDoc def = new Deflater JavaDoc (java.util.zip.Deflater.BEST_COMPRESSION);
293       DeflaterOutputStream JavaDoc dos = new DeflaterOutputStream JavaDoc (baos, def);
294       
295       ObjectOutputStream JavaDoc out = new ObjectOutputStream JavaDoc (dos);
296       out.writeObject (object);
297       out.close ();
298       dos.finish ();
299       dos.close ();
300       
301       return baos.toByteArray ();
302    }
303    
304    protected Object JavaDoc inflate (byte[] compressedContent) throws IOException JavaDoc
305    {
306       if (compressedContent==null)
307          return null;
308       
309       try
310       {
311          ObjectInputStream JavaDoc in = new ObjectInputStream JavaDoc (new InflaterInputStream JavaDoc (new ByteArrayInputStream JavaDoc (compressedContent)));
312          
313          Object JavaDoc object = in.readObject ();
314          in.close ();
315          return object;
316       }
317       catch (Exception JavaDoc e)
318       {
319          throw new IOException JavaDoc (e.toString ());
320       }
321    }
322    
323    protected Hashtable JavaDoc appSessions = new Hashtable JavaDoc ();
324    protected Object JavaDoc lockAppSession = new Object JavaDoc ();
325    
326    protected Hashtable JavaDoc getHashtableForApp (String JavaDoc appName)
327    {
328       if (this.appSessions == null)
329          this.appSessions = new Hashtable JavaDoc (); // should never happen though...
330

331       Hashtable JavaDoc result = null;
332       
333       synchronized (this.lockAppSession)
334       {
335          result = (Hashtable JavaDoc)this.appSessions.get (appName);
336          if (result == null)
337          {
338             result = new Hashtable JavaDoc ();
339             this.appSessions.put (appName, result);
340          }
341       }
342       return result;
343    }
344    
345    public void createSession (String JavaDoc appName, Object JavaDoc keyId)
346    {
347       this._createSession (appName, keyId);
348    }
349    
350    public PackagedSessionImpl _createSession (String JavaDoc appName, Object JavaDoc keyId)
351    {
352       Hashtable JavaDoc app = this.getHashtableForApp (appName);
353       PackagedSessionImpl result = new PackagedSessionImpl ((Serializable JavaDoc)keyId, null, this.myNodeName);
354       app.put (keyId, result);
355       return result;
356    }
357    
358    public void setState (String JavaDoc appName, Object JavaDoc keyId, byte[] state)
359       throws java.rmi.RemoteException JavaDoc
360    {
361       Hashtable JavaDoc app = this.getHashtableForApp (appName);
362       PackagedSession ps = (PackagedSession)app.get (keyId);
363       
364       if (ps == null)
365       {
366          ps = _createSession (appName, keyId);
367       }
368             
369       boolean isStateIdentical = false;
370       
371       Mutex mtx = getLock (appName, keyId);
372       try {
373          if (!mtx.attempt (0))
374             throw new java.rmi.RemoteException JavaDoc ("Concurent calls on session object.");
375       }
376       catch (InterruptedException JavaDoc ie) { log.info (ie); return; }
377       
378       try
379       {
380          isStateIdentical = ps.setState(state);
381          if (!isStateIdentical)
382          {
383             Object JavaDoc[] args =
384                {appName, ps};
385             try
386             {
387                this.hapGeneral.callMethodOnCluster (this.sessionStateIdentifier,
388                                                     "_setState",
389                                                     args,
390                                                     new Class JavaDoc[]{String JavaDoc.class, PackagedSession.class}, true);
391             }
392             catch (Exception JavaDoc e)
393             {
394                log.error("operation failed", e);
395             }
396          }
397       }
398       finally
399       {
400          mtx.release ();
401       }
402    }
403    
404    /*
405    public void _setStates (String appName, Hashtable packagedSessions)
406    {
407       synchronized (this.lockAppSession)
408       {
409          Hashtable app = this.getHashtableForApp (appName);
410          
411          if (app == null)
412          {
413             app = new Hashtable (packagedSessions.size ());
414             this.appSessions.put (appName, app);
415          }
416          app.putAll (packagedSessions);
417       }
418    }*/

419    
420    public void _setState (String JavaDoc appName, PackagedSession session)
421    {
422       Hashtable JavaDoc app = this.getHashtableForApp (appName);
423       PackagedSession ps = (PackagedSession)app.get (session.getKey ());
424       
425       if (ps == null)
426       {
427          ps = session;
428          synchronized (app)
429          {
430             app.put (ps.getKey (), ps);
431          }
432       }
433       else
434       {
435          Mutex mtx = getLock (appName, session.getKey ());
436          try { mtx.acquire (); } catch (InterruptedException JavaDoc ie) { log.info (ie); return; }
437          
438          try
439          {
440             if (ps.getOwner ().equals (this.myNodeName))
441             {
442                // a modification has occured externally while we were the owner
443
//
444
ownedObjectExternallyModified (appName, session.getKey (), ps, session);
445             }
446             ps.update (session);
447          }
448          finally
449          {
450             mtx.release ();
451          }
452       }
453       
454    }
455    
456    public PackagedSession getState (String JavaDoc appName, Object JavaDoc keyId)
457    {
458       Hashtable JavaDoc app = this.getHashtableForApp (appName);
459       return (PackagedSession)app.get (keyId);
460    }
461    
462    public PackagedSession getStateWithOwnership (String JavaDoc appName, Object JavaDoc keyId) throws java.rmi.RemoteException JavaDoc
463    {
464       return this.localTakeOwnership (appName, keyId);
465    }
466    
467    public PackagedSession localTakeOwnership (String JavaDoc appName, Object JavaDoc keyId) throws java.rmi.RemoteException JavaDoc
468    {
469       Hashtable JavaDoc app = this.getHashtableForApp (appName);
470       PackagedSession ps = (PackagedSession)app.get (keyId);
471       
472       // if the session is not yet available, we simply return null. The persistence manager
473
// will have to take an action accordingly
474
//
475
if (ps == null)
476          return null;
477       
478       Mutex mtx = getLock (appName, keyId);
479       
480       try {
481          if (!mtx.attempt (0))
482             throw new java.rmi.RemoteException JavaDoc ("Concurent calls on session object.");
483       }
484       catch (InterruptedException JavaDoc ie) { log.info (ie); return null; }
485       
486       try
487       {
488          if (!ps.getOwner ().equals (this.myNodeName))
489          {
490             Object JavaDoc[] args =
491             {appName, keyId, this.myNodeName, new Long JavaDoc (ps.getVersion ())};
492             ArrayList JavaDoc answers = null;
493             try
494             {
495                answers = this.hapGeneral.callMethodOnCluster (this.sessionStateIdentifier,
496                                                               "_setOwnership",
497                                                               args,
498                                                               new Class JavaDoc[]{String JavaDoc.class, Object JavaDoc.class,
499                                                                           String JavaDoc.class, Long JavaDoc.class},
500                                                               true);
501             }
502             catch (Exception JavaDoc e)
503             {
504                log.error("operation failed", e);
505             }
506             
507             if (answers != null && answers.contains (Boolean.FALSE))
508                throw new java.rmi.RemoteException JavaDoc ("Concurent calls on session object.");
509             else
510             {
511                ps.setOwner (this.myNodeName);
512                return ps;
513             }
514          }
515          else
516             return ps;
517       }
518       finally
519       {
520          mtx.release ();
521       }
522    }
523    
524    public Boolean JavaDoc _setOwnership (String JavaDoc appName, Object JavaDoc keyId, String JavaDoc newOwner, Long JavaDoc remoteVersion)
525    {
526       Hashtable JavaDoc app = this.getHashtableForApp (appName);
527       PackagedSession ps = (PackagedSession)app.get (keyId);
528       Boolean JavaDoc answer = Boolean.TRUE;
529       Mutex mtx = getLock (appName, keyId);
530       
531       try {
532          if (!mtx.attempt (0))
533             return Boolean.FALSE;
534       }
535       catch (InterruptedException JavaDoc ie) { log.info (ie); return Boolean.FALSE; }
536
537       try
538       {
539          if (!ps.getOwner ().equals (this.myNodeName))
540          {
541             // this is not our business... we don't care
542
// we do not update the owner of ps as another host may refuse the _setOwnership call
543
// anyway, the update will be sent to us later if state is modified
544
//
545
//ps.setOwner (newOwner);
546
answer = Boolean.TRUE;
547          }
548          else if (ps.getVersion () > remoteVersion.longValue ())
549          {
550             // we are concerned and our version is more recent than the one of the remote host!
551
// it means that we have concurrent calls on the same state that has not yet been updated
552
// this means we will need to raise a java.rmi.RemoteException
553
//
554
answer = Boolean.FALSE;
555          }
556          else
557          {
558             // the remote host has the same version as us (or more recent? possible?)
559
// we need to update the ownership. We can do this because we know that no other
560
// node can refuse the _setOwnership call
561
ps.setOwner (newOwner);
562             ownedObjectExternallyModified (appName, keyId, ps, ps);
563             answer = Boolean.TRUE;
564          }
565       }
566       finally
567       {
568          mtx.release ();
569       }
570       return answer;
571    }
572    
573    public void takeOwnership (String JavaDoc appName, Object JavaDoc keyId) throws java.rmi.RemoteException JavaDoc
574    {
575       this.localTakeOwnership (appName, keyId);
576    }
577    
578    public void removeSession (String JavaDoc appName, Object JavaDoc keyId)
579    {
580       Hashtable JavaDoc app = this.getHashtableForApp (appName);
581       if (app != null)
582       {
583          PackagedSession ps = (PackagedSession)app.remove (keyId);
584          if (ps != null)
585          {
586             removeLock (appName, keyId);
587             Object JavaDoc[] args =
588                { appName, keyId };
589             try
590             {
591                this.hapGeneral.callMethodOnCluster (this.sessionStateIdentifier,
592                                                     "_removeSession",
593                                                     args,
594                                                     new Class JavaDoc[]{String JavaDoc.class, Object JavaDoc.class},
595                                                     true);
596             }
597             catch (Exception JavaDoc e)
598             { log.error("operation failed", e); }
599          }
600       }
601    }
602    
603    public void _removeSession (String JavaDoc appName, Object JavaDoc keyId)
604    {
605       Hashtable JavaDoc app = this.getHashtableForApp (appName);
606       PackagedSession ps = null;
607       ps = (PackagedSession)app.remove (keyId);
608       if (ps != null && ps.getOwner ().equals (this.myNodeName))
609          ownedObjectExternallyModified (appName, keyId, ps, ps);
610       
611       removeLock (appName, keyId);
612    }
613    
614    protected Hashtable JavaDoc listeners = new Hashtable JavaDoc ();
615    
616    public synchronized void subscribe (String JavaDoc appName, HASessionStateListener listener)
617    {
618       Vector JavaDoc members = (Vector JavaDoc)listeners.get (appName);
619       if (members == null)
620       {
621          members = new Vector JavaDoc ();
622          listeners.put (appName, members);
623       }
624       if (!members.contains (listener))
625       {
626          members.add (listener);
627       }
628
629    }
630    
631    public synchronized void unsubscribe (String JavaDoc appName, HASessionStateListener listener)
632    {
633       Vector JavaDoc members = (Vector JavaDoc)listeners.get (appName);
634       if ((members != null) && members.contains (listener))
635          members.remove (listener);
636    }
637    
638    public void ownedObjectExternallyModified (String JavaDoc appName, Object JavaDoc key, PackagedSession oldSession, PackagedSession newSession)
639    {
640       Vector JavaDoc members = (Vector JavaDoc)listeners.get (appName);
641       if (members != null)
642          for (int i=0; i<members.size (); i++)
643          try
644          {
645             ((HASessionStateListener)members.elementAt (i)).sessionExternallyModified (newSession);
646          }
647          catch (Throwable JavaDoc t)
648          {
649             log.debug (t);
650          }
651    }
652    
653    public HAPartition getCurrentHAPartition ()
654    {
655       return this.hapGeneral;
656    }
657    
658    
659    protected boolean lockExists (String JavaDoc appName, Object JavaDoc key)
660    {
661       synchronized (this.locks)
662       {
663          HashMap JavaDoc ls = (HashMap JavaDoc)this.locks.get (appName);
664          if (ls == null)
665             return false;
666          
667          return (ls.get(key)!=null);
668       }
669    }
670
671    protected Mutex getLock (String JavaDoc appName, Object JavaDoc key)
672    {
673       synchronized (this.locks)
674       {
675          HashMap JavaDoc ls = (HashMap JavaDoc)this.locks.get (appName);
676          if (ls == null)
677          {
678             ls = new HashMap JavaDoc ();
679             this.locks.put (appName, ls);
680          }
681           
682          Mutex mutex = (Mutex)ls.get(key);
683          if (mutex == null)
684          {
685             mutex = new Mutex ();
686             ls.put (key, mutex);
687          }
688          
689          return mutex;
690       }
691    }
692
693    protected void removeLock (String JavaDoc appName, Object JavaDoc key)
694    {
695       synchronized (this.locks)
696       {
697          HashMap JavaDoc ls = (HashMap JavaDoc)this.locks.get (appName);
698          if (ls == null)
699             return;
700          ls.remove (key);
701       }
702    }
703    
704 }
705
Popular Tags