RepositoryUpdater


1   /*
2    * The contents of this file are subject to the terms of the Common Development
3    * and Distribution License (the License). You may not use this file except in
4    * compliance with the License.
5    *
6    * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7    * or http://www.netbeans.org/cddl.txt.
8    *
9    * When distributing Covered Code, include this CDDL Header Notice in each file
10   * and include the License file at http://www.netbeans.org/cddl.txt.
11   * If applicable, add the following below the CDDL Header, with the fields
12   * enclosed by brackets [] replaced by your own identifying information:
13   * "Portions Copyrighted [year] [name of copyright owner]"
14   *
15   * The Original Software is NetBeans. The Initial Developer of the Original
16   * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
17   * Microsystems, Inc. All Rights Reserved.
18   */
19  
20  package org.netbeans.modules.retouche.source.usages;
21  
22  import java.beans.PropertyChangeEvent  ;
23  import java.beans.PropertyChangeListener  ;
24  import java.io.BufferedReader  ;
25  import java.io.File  ;
26  import java.io.FileReader  ;
27  import java.io.IOException  ;
28  import java.net.URI  ;
29  import java.net.URL  ;
30  import java.util.ArrayList  ;
31  import java.util.Collections  ;
32  import java.util.HashMap  ;
33  import java.util.HashSet  ;
34  import java.util.Iterator  ;
35  import java.util.LinkedList  ;
36  import java.util.List  ;
37  import java.util.Map  ;
38  import java.util.RandomAccess  ;
39  import java.util.Set  ;
40  import java.util.Stack  ;
41  import java.util.StringTokenizer  ;
42  import java.util.Timer  ;
43  import java.util.TimerTask  ;
44  import java.util.concurrent.CountDownLatch  ;
45  import java.util.concurrent.atomic.AtomicBoolean  ;
46  import java.util.logging.Level  ;
47  import java.util.logging.Logger  ;
48  import javax.swing.event.ChangeEvent  ;
49  import javax.swing.event.ChangeListener  ;
50  import org.netbeans.api.gsf.Error;
51  import org.netbeans.api.gsf.Severity;
52  import org.netbeans.api.gsf.Indexer;
53  import org.netbeans.api.gsf.ParseEvent;
54  import org.netbeans.api.gsf.ParseListener;
55  import org.netbeans.api.gsf.ParserFile;
56  import org.netbeans.api.gsf.ParserResult;
57  import org.netbeans.api.gsf.CancellableTask;
58  import org.netbeans.api.java.classpath.ClassPath;
59  import org.netbeans.api.java.queries.SourceLevelQuery;
60  import org.netbeans.api.progress.ProgressHandle;
61  import org.netbeans.api.progress.ProgressHandleFactory;
62  import org.netbeans.api.queries.VisibilityQuery;
63  import org.netbeans.api.retouche.source.ClasspathInfo;
64  import org.netbeans.api.retouche.source.CompilationInfo;
65  import org.netbeans.api.retouche.source.ParserTaskImpl;
66  import org.netbeans.api.retouche.source.Source;
67  import org.netbeans.modules.gsf.Language;
68  import org.netbeans.modules.gsf.LanguageRegistry;
69  import org.netbeans.modules.retouche.source.CacheClassPath;
70  import org.netbeans.modules.retouche.source.GlobalSourcePath;
71  import org.netbeans.modules.retouche.source.SourceAccessor;
72  import org.netbeans.modules.retouche.source.SourceAccessor;
73  import org.netbeans.modules.retouche.source.parsing.FileObjects;
74  import org.netbeans.modules.retouche.source.usages.ClassIndexManager;
75  import org.netbeans.modules.retouche.source.util.LowMemoryEvent;
76  import org.netbeans.modules.retouche.source.util.LowMemoryListener;
77  import org.netbeans.modules.retouche.source.util.LowMemoryNotifier;
78  import org.netbeans.spi.java.classpath.ClassPathFactory;
79  import org.openide.filesystems.FileAttributeEvent;
80  import org.openide.filesystems.FileChangeListener;
81  import org.openide.filesystems.FileEvent;
82  import org.openide.filesystems.FileObject;
83  import org.openide.filesystems.FileRenameEvent;
84  import org.openide.filesystems.FileStateInvalidException;
85  import org.openide.filesystems.FileSystem;
86  import org.openide.filesystems.FileUtil;
87  import org.openide.filesystems.URLMapper;
88  import org.openide.util.Exceptions;
89  import org.openide.util.NbBundle;
90  import org.openide.util.TopologicalSortException;
91  import org.openide.util.Utilities;
92  
93  /**
94   * RepositoryUpdater is in charge of maintaining indices of the various classes in 
95   * the system, for use by code completion, go to declaration, etc.  The classes include
96   * not only the user's source and test directories, but jars from the boot class path etc.
97   * The RepositoryUpdater schedules indexing jobs, watches filesystems for modifications,
98   * determines whether an index is out of date, etc.
99   * 
100  * This class is originally from Retouche, under the java/source module. Since it's
101  * an important and fairly complicated piece of logic, I am trying my best to keep
102  * my copy in sync with the java one. Therefore, I have left the original formatting
103  * in place as much as possible. Please don't make gratuitous formatting changes that
104  * makes diffing harder.
105  * 
106  * There are some important changes. Obviously, the various javac-specific setup code
107  * has changed, and I also need to -iterate- over files to be indexed to let the
108  * potentially multiple language indexers each have a chance to index the file.
109  * I have also changed references to other parts that have been renamed, such
110  * as JavaSource => Source, etc.
111  * 
112  * @author Tomas Zezula
113  * @author Tor Norbye
114  */
115 public class RepositoryUpdater implements PropertyChangeListener  , FileChangeListener {
116     private static final boolean PREINDEXING = Boolean.getBoolean("gsf.preindexing");
117     private static final Logger   LOGGER = Logger.getLogger(RepositoryUpdater.class.getName());
118     private static final Set  <String  > ignoredDirectories = parseSet("org.netbeans.javacore.ignoreDirectories", "SCCS CVS .svn"); // NOI18N
119     private static final boolean noscan = Boolean.getBoolean("netbeans.javacore.noscan");   //NOI18N
120     public static final boolean PERF_TEST = Boolean.getBoolean("perf.refactoring.test");
121     //private static final String PACKAGE_INFO = "package-info.java";  //NOI18N
122     
123     private static final int DELAY = Utilities.isWindows() ? 2000 : 1000;
124     
125     private static RepositoryUpdater instance;
126     
127     private final GlobalSourcePath cpImpl;
128     private final ClassPath cp;
129     private final ClassPath ucp;    
130     private final ClassPath binCp;
131     private Set  <URL  > scannedRoots;
132     private Set  <URL  > scannedBinaries;
133     private Delay delay;
134     private Work currentWork;
135     private boolean dirty;
136     private int noSubmited;
137     
138     //Preprocessor support
139     //private final Map<URL, JavaFileFilterImplementation> filters = Collections.synchronizedMap(new HashMap<URL, JavaFileFilterImplementation>());
140     private final FilterListener filterListener = new FilterListener ();    
141     
142     /** Creates a new instance of RepositoryUpdater */
143     private RepositoryUpdater() {
144         this.delay = new Delay();
145         this.cpImpl = GlobalSourcePath.getDefault();
146         this.cp = ClassPathFactory.createClassPath (this.cpImpl.getSourcePath());
147         this.cp.addPropertyChangeListener(this);
148         this.ucp = ClassPathFactory.createClassPath (this.cpImpl.getUnknownSourcePath());
149         this.binCp = ClassPathFactory.createClassPath(this.cpImpl.getBinaryPath());
150         this.registerFileSystemListener();
151         this.scannedRoots = Collections.synchronizedSet(new HashSet  <URL  >());
152         this.scannedBinaries = Collections.synchronizedSet(new HashSet  <URL  >());
153         submitBatch();
154     }
155     
156     public ClassPath getScannedSources () {
157         return this.cp;
158     }
159     
160     public ClassPath getScannedBinaries() {
161         return this.binCp;
162     }
163     
164     public void close () {                
165         this.cp.removePropertyChangeListener(this);
166         this.unregisterFileSystemListener();
167         this.delay.cancel();
168     }
169 
170     public void propertyChange(PropertyChangeEvent   evt) {
171         if (ClassPath.PROP_ROOTS.equals(evt.getPropertyName())) {
172             submitBatch();
173         }
174     }
175     
176     private synchronized void submitBatch () {
177         if (this.currentWork == null) {                    
178             this.currentWork = Work.batch();
179             submit (this.currentWork);
180         }
181         else {
182             this.dirty = true;
183         }
184     }
185     
186     public synchronized void waitScanFinished () throws InterruptedException   {
187         while (this.noSubmited > 0 ) {
188             this.wait();
189         }
190     }
191     
192     
193     private synchronized boolean isDirty () {
194         if (this.dirty) {
195             this.dirty = false;
196             return true;
197         }
198         else {
199             this.currentWork = null;
200             return false;                        
201         }
202     }
203     
204     private synchronized void resetDirty () {
205         this.dirty = false;
206         this.currentWork = null;
207     }
208     
209     
210     public void fileRenamed(FileRenameEvent fe) {
211         final FileObject fo = fe.getFile();
212         try {
213             if ((isRelevantSource (fo) || fo.isFolder()) && VisibilityQuery.getDefault().isVisible(fo)) {
214                 final URL   root = getOwningSourceRoot(fo);
215                 if (root != null) {                
216                     String   originalName = fe.getName();
217                     final String   originalExt = fe.getExt();
218                     if (originalExt.length()>0) {
219                         originalName = originalName+'.'+originalExt;  //NOI18N
220                     }
221                     final File   parentFile = FileUtil.toFile(fo.getParent());
222                     if (parentFile != null) {
223                         final URL   original = new File   (parentFile,originalName).toURI().toURL();
224                         submit(Work.delete(original,root,fo.isFolder()));
225                         delay.post(Work.compile (fo,root));
226                     }
227                 }
228             }
229             else if (isBinary(fo) && VisibilityQuery.getDefault().isVisible(fo)) {
230                 final URL   root = getOwningBinaryRoot(fo);
231                 if (root != null) {
232                     String   originalName = fe.getName();
233                     final String   originalExt = fe.getExt();
234                     if (originalExt.length()>0) {
235                         originalName = originalName+'.'+originalExt;    //NOI18N
236                     }
237                     final File   parentFile = FileUtil.toFile(fo.getParent());
238                     if (parentFile != null) {
239                         final URL   original = new File   (parentFile,originalName).toURI().toURL();
240                         submit(Work.binary(original, root, fo.isFolder()));
241                         submit(Work.binary(fo, root));
242                     }
243                 }
244             }
245         } catch (IOException   ioe) {
246             Exceptions.printStackTrace(ioe);
247         }
248     }
249 
250     public void fileAttributeChanged(FileAttributeEvent fe) {
251         //Not interesting, do nothing
252     }
253 
254     public void fileFolderCreated(FileEvent fe) {
255         //In ideal case this should do nothing,
256         //but in Netbeans newly created folder may
257         //already contain files
258         final FileObject fo = fe.getFile();
259         try {
260             final URL   root = getOwningSourceRoot(fo);
261             if ( root != null && VisibilityQuery.getDefault().isVisible(fo)) {
262                 scheduleCompilation(fo,root);
263             }
264         } catch (IOException   ioe) {
265             Exceptions.printStackTrace(ioe);
266         }
267     }
268 
269     public void fileDeleted(FileEvent fe) {
270         final FileObject fo = fe.getFile();        
271         final boolean isFolder = fo.isFolder();
272         try {
273             if ((isRelevantSource(fo) || isFolder) && VisibilityQuery.getDefault().isVisible(fo)) {
274                 final URL   root = getOwningSourceRoot (fo);
275                 if (root != null) {                
276                     submit(Work.delete(fo,root,isFolder));
277                 }
278             }
279             else if ((isBinary(fo) || isFolder) && VisibilityQuery.getDefault().isVisible(fo)) {
280                 final URL   root = getOwningBinaryRoot(fo);
281                 if (root !=null) {
282                     submit(Work.binary(fo,root));
283                 }
284             }
285         } catch (IOException   ioe) {
286             Exceptions.printStackTrace(ioe);
287         }
288     }
289 
290     public void fileDataCreated(FileEvent fe) {
291         final FileObject fo = fe.getFile();        
292         try {
293             if (isRelevantSource(fo) && VisibilityQuery.getDefault().isVisible(fo)) {
294                 final URL   root = getOwningSourceRoot (fo);        
295                 if (root != null) {
296                     postCompilation(fo, root);
297                 }
298             }
299             else if (isBinary(fo) && VisibilityQuery.getDefault().isVisible(fo)) {
300                 final URL   root = getOwningBinaryRoot(fo);
301                 if (root != null) {
302                     submit(Work.binary(fo, root));
303                 }
304             }
305         } catch (IOException   ioe) {
306             Exceptions.printStackTrace(ioe);
307         }
308     }
309 
310     public void fileChanged(FileEvent fe) {
311         final FileObject fo = fe.getFile();
312         try {
313             if (isRelevantSource(fo) && VisibilityQuery.getDefault().isVisible(fo)) {
314                 final URL   root = getOwningSourceRoot (fo);
315                 if (root != null) {
316                     postCompilation(fo, root);
317                 }
318             }        
319             else if (isBinary(fo) && VisibilityQuery.getDefault().isVisible(fo)) {
320                 final URL   root = getOwningBinaryRoot(fo);
321                 if (root != null) {
322                     submit(Work.binary(fo, root));
323                 }
324             }
325         } catch (IOException   ioe) {
326             Exceptions.printStackTrace(ioe);
327         }
328     }  
329     
330     
331     public final void scheduleCompilation (final FileObject fo, final FileObject root) throws IOException   {
332         URL   foURL = fo.getURL();
333         URL   rootURL = root.getURL();
334         assert "file".equals(foURL.getProtocol()) && "file".equals(rootURL.getProtocol());
335         scheduleCompilation (foURL,rootURL,fo.isFolder());
336     }      
337     
338     private final void scheduleCompilation (final FileObject fo, final URL   root) throws IOException   {
339         scheduleCompilation (fo.getURL(),root,fo.isFolder());
340     }
341     
342     private final void scheduleCompilation (final URL   file, final URL   root, boolean isFolder) {
343         submit(Work.compile (file,root, isFolder));
344     }
345     
346     private final void postCompilation (final FileObject file, final URL   root) throws FileStateInvalidException {
347         delay.post (Work.compile (file,root));
348     }
349     
350     
351     /**
352      * This method is only for unit tests.
353      * Test can schedule compilation and wait on the returned {@link CountDownLatch}
354      * until the compilation is finished.
355      * @param folder to be compiled
356      * @param root the source root. The folder has to be either under the root or
357      * equal to the root.
358      * @return {@link CountDownLatch} to wait on.
359      */
360     public final CountDownLatch   scheduleCompilationAndWait (final FileObject folder, final FileObject root) throws IOException   {
361         CountDownLatch  [] latch = new CountDownLatch  [1];
362         submit(Work.compile (folder,root.getURL(),latch));
363         return latch[0];
364     }
365     
366     private void submit (final Work  work) {
367         if (!noscan) {
368             synchronized (this) {
369                 this.noSubmited++;
370             }
371             final CompileWorker cw = new CompileWorker (work);
372             SourceAccessor.INSTANCE.runSpecialTask (cw, Source.Priority.MAX);
373         }
374     }
375     
376     
377     private void registerFileSystemListener  () {
378         final File  [] roots = File.listRoots();
379         final Set  <FileSystem> fss = new HashSet  <FileSystem> ();
380         for (File   root : roots) {
381             final FileObject fo = FileUtil.toFileObject (root);
382             if (fo == null) {
383                 Logger.getLogger("global").warning("No MasterFS for file system root: " + root.getAbsolutePath());          // NOI18N
384             }
385             else {
386                 try {
387                     final FileSystem fs = fo.getFileSystem();
388                     if (!fss.contains(fs)) {
389                         fs.addFileChangeListener (this);
390                         fss.add(fs);
391                     }
392                 } catch (FileStateInvalidException e) {
393                     Exceptions.printStackTrace(e);
394                 }
395             }
396         }
397     }
398     
399     private void unregisterFileSystemListener () {
400         final File  [] roots = File.listRoots();
401         final Set  <FileSystem> fss = new HashSet  <FileSystem> ();
402         for (File   root : roots) {
403             final FileObject fo = FileUtil.toFileObject (root);
404             if (fo == null) {
405                 Logger.getLogger("global").warning("No MasterFS for file system root: " + root.getAbsolutePath());          // NOI18N
406             }
407             else {
408                 try {
409                     final FileSystem fs = fo.getFileSystem();
410                     if (!fss.contains(fs)) {
411                         fs.removeFileChangeListener (this);
412                         fss.add(fs);
413                     }
414                 } catch (FileStateInvalidException e) {
415                     Exceptions.printStackTrace(e);
416                 }
417             }
418         }
419     }
420     
421     private URL   getOwningSourceRoot (final FileObject fo) {
422         if (fo == null) {
423             return null;
424         }
425         synchronized (this.scannedRoots) {
426             for (URL   root : this.scannedRoots) {
427                 FileObject rootFo = URLMapper.findFileObject(root);
428                 if (rootFo != null && FileUtil.isParentOf(rootFo,fo)) {
429                     return root;
430                 }
431             }
432         }
433         return null;
434     }        
435     
436     private URL   getOwningBinaryRoot (final FileObject fo){
437         if (fo == null) {
438             return null;
439         }
440         try {
441             synchronized (this.scannedBinaries) {
442                 URL   foURL = fo.getURL();
443                 for (URL   root : this.scannedBinaries) {
444                     URL   fileURL = FileUtil.getArchiveFile(root);
445                     boolean archive = true;
446                     if (fileURL == null) {
447                         fileURL = root;
448                         archive = false;
449                     }
450                     String   filePath = fileURL.getPath();
451                     String   foPath = foURL.getPath();
452                     if (filePath.equals(foPath)) {
453                         return root;
454                     }                    
455                     if (!archive && foPath.startsWith(filePath)) {                        
456                         return root;
457                     }
458                 }
459             }
460         } catch (FileStateInvalidException fsi) {
461             Exceptions.printStackTrace(fsi);
462         }
463         return null;
464     }
465     
466     /**
467      * Temporary implementation which does not care about
468      * extended mime types like text/x-something+x-java
469      */
470     private static boolean isRelevantSource (final FileObject fo) {
471         if (fo.isFolder()) {
472             return false;
473         }
474 
475         if (LanguageRegistry.getInstance().isSupported(fo.getMIMEType())) {
476             return true;
477         }
478         
479         return false;
480     }
481     
482     private static boolean isBinary (final FileObject fo) {
483         return false;
484         /* XXX TODO no support for binary persistence files yet, such as compiled ruby files etc.
485         if (fo.isFolder()) {
486             return false;
487         }
488         String ext = fo.getExt().toLowerCase();
489         if (FileObjects.CLASS.equals(ext) ||
490             FileObjects.JAR.equals(ext) ||
491             FileObjects.ZIP.equals(ext)) { 
492                 return true;
493         }        
494         return false;
495          */
496     }
497     
498     private static enum WorkType {
499         COMPILE_BATCH, COMPILE_CONT, COMPILE, DELETE, UPDATE_BINARY, FILTER_CHANGED
500     };
501     
502     
503     private static class Work {
504         private final WorkType workType;
505         private final CountDownLatch   latch;
506         
507         protected Work (WorkType workType, CountDownLatch   latch) {
508             assert workType != null;
509             this.workType = workType;
510             this.latch = latch;
511         }
512         
513         public WorkType getType () {
514             return this.workType;
515         }
516         
517         public void finished () {
518             if (this.latch != null) {
519                 this.latch.countDown();
520             }
521         }
522         
523         
524         public static Work batch () {
525             return new Work (WorkType.COMPILE_BATCH, null);
526         }
527         
528         public static Work compile (final FileObject file, final URL   root) throws FileStateInvalidException {
529             return compile (file.getURL(), root, file.isFolder());
530         }
531         
532         public static Work compile (final URL   file, final URL   root, boolean isFolder) {
533             assert file != null && root != null;
534             return new SingleRootWork (WorkType.COMPILE, file, root, isFolder, null);
535         }
536         
537         public static Work compile (final FileObject file, final URL   root, CountDownLatch  [] latch) throws FileStateInvalidException {
538             assert file != null && root != null;
539             assert latch != null && latch.length == 1 && latch[0] == null;
540             latch[0] = new CountDownLatch   (1);
541             return new SingleRootWork (WorkType.COMPILE, file.getURL(), root, file.isFolder(),latch[0]);
542         }
543         
544         public static Work delete (final FileObject file, final URL   root, final boolean isFolder) throws FileStateInvalidException {
545             return delete (file.getURL(), root,file.isFolder());
546         }
547         
548         public static Work delete (final URL   file, final URL   root, final boolean isFolder) {
549             assert file != null && root != null;
550             return new SingleRootWork (WorkType.DELETE, file, root, isFolder,null);
551         }
552         
553         public static Work binary (final FileObject file, final URL   root) throws FileStateInvalidException {
554             return binary (file.getURL(), root, file.isFolder());
555         }
556         
557         public static Work binary (final URL   file, final URL   root, boolean isFolder) {
558             assert file != null && root != null;
559             return new SingleRootWork (WorkType.UPDATE_BINARY, file, root, isFolder, null);
560         }
561         
562         public static Work filterChange (final List  <URL  > roots) {
563             assert roots != null;
564             return new MultiRootsWork (WorkType.FILTER_CHANGED, roots, null);
565         }
566         
567     }
568         
569     private static class SingleRootWork extends Work {
570         
571         private URL   file;
572         private URL   root;
573         private boolean isFolder;
574         
575                 
576         public SingleRootWork (WorkType type, URL   file, URL   root, boolean isFolder, CountDownLatch   latch) {
577             super (type, latch);           
578             this.file = file;            
579             this.root = root;
580             this.isFolder = isFolder;
581         }
582         
583         public URL   getFile () {
584             return this.file;
585         }
586         
587         public URL   getRoot () {
588             return this.root;
589         }
590         
591         public boolean isFolder () {            
592             return this.isFolder;            
593         }
594         
595     }
596     
597     private static class MultiRootsWork extends Work {
598         private List  <URL  > roots;
599         
600         public MultiRootsWork (WorkType type, List  <URL  > roots, CountDownLatch   latch) {
601             super (type, latch);
602             this.roots = roots;
603         }
604         
605         public List  <URL  > getRoots () {
606             return roots;
607         }
608     }          
609     
610     private final class CompileWorker implements CancellableTask<CompilationInfo> {
611                 
612         private Work work;
613         private List  <URL  > state;
614         private Set  <URL  > oldRoots;
615         private Set  <URL  > oldBinaries;
616         private Set  <URL  > newBinaries;
617         private ProgressHandle handle;
618         private final Set  <URI  > dirtyCrossFiles;
619         private final AtomicBoolean   canceled;
620         
621         public CompileWorker (Work work ) {
622             assert work != null;
623             this.work = work;
624             this.canceled = new AtomicBoolean   (false);
625             this.dirtyCrossFiles = new HashSet  <URI  >();
626         }
627         
628         public void cancel () {            
629             this.canceled.set(true);
630         }
631         
632         public void run (final CompilationInfo nullInfo) throws IOException   {
633             ClassIndexManager.getDefault().writeLock (new ClassIndexManager.ExceptionAction<Void  > () {
634                 
635                 @SuppressWarnings  ("fallthrough")
636                 public Void   run () throws IOException   {
637                     boolean continuation = false;
638                     try {
639                     final WorkType type = work.getType();                        
640                     switch (type) {
641                         case FILTER_CHANGED:                            
642                             try {
643                                 final MultiRootsWork mw = (MultiRootsWork) work;
644                                 final List  <URL  > roots = mw.getRoots();
645                                 final Map  <URL  ,List  <URL  >> depGraph = new HashMap  <URL  ,List  <URL  >> ();
646                                 for (URL   root: roots) {
647                                     findDependencies (root, new Stack  <URL  >(), depGraph, null, false);
648                                 }
649                                 state = Utilities.topologicalSort(roots, depGraph);                                                                             
650                                 for (java.util.ListIterator  <URL  > it = state.listIterator(state.size()); it.hasPrevious(); ) {                
651                                     final URL   rootURL = it.previous();
652                                     it.remove();
653                                     updateFolder (rootURL,rootURL, true, handle);
654                                 }                                
655                             } catch (final TopologicalSortException tse) {
656                                     final IllegalStateException   ise = new IllegalStateException   ();                                
657                                     throw (IllegalStateException  ) ise.initCause(tse);
658                             }
659                             break;
660                         case COMPILE_BATCH:
661                         {
662                             assert handle == null;
663                             handle = ProgressHandleFactory.createHandle(NbBundle.getMessage(RepositoryUpdater.class,"MSG_BackgroundCompileStart"));
664                             handle.start();
665                             boolean completed = false;
666                             try {
667                                 oldRoots = new HashSet  <URL  > (scannedRoots);
668                                 oldBinaries = new HashSet  <URL  > (scannedBinaries);
669                                 final List  <ClassPath.Entry> entries = new LinkedList  <ClassPath.Entry>();
670                                 entries.addAll (cp.entries());
671                                 entries.addAll (ucp.entries());
672                                 final List  <ClassPath.Entry> binaryEntries = binCp.entries();
673                                 newBinaries = new HashSet  <URL  > ();
674                                 for (ClassPath.Entry entry : binaryEntries) {
675                                     URL   binRoot = entry.getURL();
676                                     if (!oldBinaries.remove(binRoot)) {
677                                         newBinaries.add (binRoot);
678                                     }
679                                 }
680                                 final Map  <URL  ,List  <URL  >> depGraph = new HashMap  <URL  ,List  <URL  >> ();
681                                 for (ClassPath.Entry entry : entries) {
682                                     final URL   rootURL = entry.getURL();
683                                     findDependencies (rootURL, new Stack  <URL  >(), depGraph, newBinaries, true);
684                                 }                                
685                                 CompileWorker.this.state = Utilities.topologicalSort(depGraph.keySet(), depGraph);
686                                 completed = true;
687                             } catch (final TopologicalSortException tse) {
688                                 final IllegalStateException   ise = new IllegalStateException   ();                                
689                                 throw (IllegalStateException  ) ise.initCause(tse);
690                             } finally {
691                                 if (!completed) {
692                                     resetDirty();
693                                 }
694                             }
695                         }
696                         case COMPILE_CONT:
697                             boolean completed = false;
698                             try {
699                                 if (!scanRoots()) {
700                                     CompileWorker.this.work = new Work (WorkType.COMPILE_CONT,null);
701                                     SourceAccessor.INSTANCE.runSpecialTask (CompileWorker.this, Source.Priority.MAX);
702                                     continuation = true;
703                                     return null;
704                                 }
705                                 while (isDirty()) {
706                                     assert CompileWorker.this.state.isEmpty();
707                                     final List  <ClassPath.Entry> entries = new LinkedList  <ClassPath.Entry>();
708                                     entries.addAll (cp.entries());
709                                     entries.addAll (ucp.entries());
710                                     final List  <ClassPath.Entry> binaryEntries = binCp.entries();
711                                     newBinaries = new HashSet  <URL  > ();
712                                     for (ClassPath.Entry entry : binaryEntries) {
713                                         URL   binRoot = entry.getURL();
714                                         if (!scannedBinaries.contains(binRoot)) {
715                                             newBinaries.add(binRoot);
716                                         }
717                                         else {
718                                             oldBinaries.remove(binRoot);
719                                         }
720                                     }
721                                     final Map  <URL  ,List  <URL  >> depGraph = new HashMap  <URL  ,List  <URL  >> ();
722                                     for (ClassPath.Entry entry : entries) {
723                                         final URL   rootURL = entry.getURL();
724                                         findDependencies (rootURL, new Stack  <URL  >(), depGraph, newBinaries, true);
725                                     }
726                                     try {
727                                         CompileWorker.this.state = Utilities.topologicalSort(depGraph.keySet(), depGraph);
728                                     } catch (final TopologicalSortException tse) {
729                                         final IllegalStateException   ise = new IllegalStateException   ();
730                                         throw (IllegalStateException  ) ise.initCause(tse);
731                                     }
732                                     if (!scanRoots ()) {
733                                         CompileWorker.this.work = new Work (WorkType.COMPILE_CONT,null);
734                                         SourceAccessor.INSTANCE.runSpecialTask (CompileWorker.this, Source.Priority.MAX);
735                                         continuation = true;
736                                         return null;
737                                     }
738                                 }
739                                 completed = true;
740                             } finally {
741                                 if (!completed && !continuation) {
742                                     resetDirty ();
743                                 }
744                             }                            
745                             final ClassIndexManager cim = ClassIndexManager.getDefault();
746                             scannedRoots.removeAll(oldRoots);
747                             for (URL   oldRoot : oldRoots) {
748                                 cim.removeRoot(oldRoot);
749 //                                JavaFileFilterImplementation filter = filters.remove(oldRoot);
750 //                                if (filter != null && !filters.values().contains(filter)) {
751 //                                    filter.removeChangeListener(filterListener);
752 //                                }
753                             }
754                             scannedBinaries.removeAll (oldBinaries);
755 //                            final CachingArchiveProvider cap = CachingArchiveProvider.getDefault();
756                             for (URL   oldRoot : oldBinaries) {
757                                 cim.removeRoot(oldRoot);
758 //                                cap.removeArchive(oldRoot);
759                             }
760                             break;
761                         case COMPILE:
762                         {
763                             try {
764                                 final SingleRootWork sw = (SingleRootWork) work;
765                                 final URL   file = sw.getFile();
766                                 final URL   root = sw.getRoot ();
767                                 if (sw.isFolder()) {
768                                     updateFolder (file, root, false, handle);
769                                 }
770                                 else {
771                                     updateFile (file,root);
772                                 }
773                             //} catch (Abort abort) {
774                             } catch (Exception   abort) {
775                                 //Ignore abort
776                             }                         
777                             break;
778                         }
779                         case DELETE:
780                         {
781                             final SingleRootWork sw = (SingleRootWork) work;
782                             final URL   file = sw.getFile();
783                             final URL   root = sw.getRoot ();
784                             delete (file, root, sw.isFolder());
785                             break;                                 
786                         }
787                         case UPDATE_BINARY:
788                         {
789                             SingleRootWork sw = (SingleRootWork) work;
790                             final URL   file = sw.getFile();
791                             final URL   root = sw.getRoot();
792                             updateBinary (file, root);
793                             break;
794                         }
795                     }                                                
796                     return null;                    
797                 } finally {
798                     if (!continuation) {
799                         synchronized (RepositoryUpdater.this) {
800                             RepositoryUpdater.this.noSubmited--;
801                             if (RepositoryUpdater.this.noSubmited == 0) {
802                                 RepositoryUpdater.this.notifyAll();
803                             }
804                         }
805                         work.finished ();
806                         if (handle != null) {
807                             handle.finish ();
808                         }
809                     }
810                 }
811             }});
812         }
813         
814         private void findDependencies (final URL   rootURL, final Stack  <URL  > cycleDetector, final Map  <URL  ,List  <URL  >> depGraph,
815             final Set  <URL  > binaries, final boolean useInitialState) {  
816             if (useInitialState && RepositoryUpdater.this.scannedRoots.contains(rootURL)) {                                
817                 this.oldRoots.remove(rootURL);                        
818                 return;
819             }
820             if (depGraph.containsKey(rootURL)) {
821                 return;
822             }                                                
823             final FileObject rootFo = URLMapper.findFileObject(rootURL);
824             if (rootFo == null) {               
825                 return;
826             }
827             
828             // BEGIN TOR MODIFICATIONS
829             // I don't want to start asking for the ClassPath of directories in the libraries
830             // since these start yielding Java jars etc.
831             if (true) {
832                 //if (useInitialState) {
833                    depGraph.put(rootURL,new LinkedList  <URL  > ());
834 
835                     if (!RepositoryUpdater.this.scannedBinaries.contains(rootURL)) {
836                         binaries.add (rootURL);
837                     }
838                     oldBinaries.remove(rootURL);
839                 //}
840                 return;
841             }
842             // END TOR MODIFICATIONS
843             
844             cycleDetector.push (rootURL);
845             final ClassPath bootPath = ClassPath.getClassPath(rootFo, ClassPath.BOOT);
846             final ClassPath compilePath = ClassPath.getClassPath(rootFo, ClassPath.COMPILE);
847             final ClassPath[] pathsToResolve = new ClassPath[] {bootPath,compilePath};
848             final List  <URL  > deps = new LinkedList  <URL  > ();
849             for (int i=0; i< pathsToResolve.length; i++) {
850                 final ClassPath pathToResolve = pathsToResolve[i];
851                 if (pathToResolve != null) {
852                     for (ClassPath.Entry entry : pathToResolve.entries()) {
853                         final URL   url = entry.getURL();
854                         final URL  [] sourceRoots = RepositoryUpdater.this.cpImpl.getSourceRootForBinaryRoot(url, pathToResolve, false);
855                         if (sourceRoots != null) {
856                             for (URL   sourceRoot : sourceRoots) {
857                                 if (!cycleDetector.contains(sourceRoot)) {
858                                     deps.add (sourceRoot);
859                                     findDependencies(sourceRoot, cycleDetector,depGraph, binaries, useInitialState);
860                                 }
861                             }
862                         }
863                         else {
864                             if (useInitialState) {
865                                 if (!RepositoryUpdater.this.scannedBinaries.contains(url)) {
866                                     binaries.add (url);
867                                 }
868                                 oldBinaries.remove(url);
869                             }
870                         }
871                     }
872                 }
873             }
874             depGraph.put(rootURL,deps);
875             cycleDetector.pop ();
876         }
877         
878         private boolean scanRoots () {
879             
880             for (Iterator  <URL  > it = this.newBinaries.iterator(); it.hasNext(); ) {
881                 if (this.canceled.getAndSet(false)) {                    
882                     return false;
883                 }
884                 final URL   rootURL = it.next();
885                 try {
886                     it.remove();
887                     final ClassIndexImpl ci = ClassIndexManager.getDefault().createUsagesQuery(rootURL,false);
888                     final File   rootFile = FileObjects.getRootFile(rootURL);
889                     final String   message = String.format (NbBundle.getMessage(RepositoryUpdater.class,"MSG_Scannig"),rootFile.getAbsolutePath());
890                     handle.setDisplayName(message);
891                     RepositoryUpdater.this.scannedBinaries.add (rootURL);
892                     if (rootFile.canRead()) {                        
893                         long startT = System.currentTimeMillis();
894                         ci.getBinaryAnalyser().analyse(rootFile, handle);
895                         long endT = System.currentTimeMillis();
896                         if (PERF_TEST) {
897                             try {
898                                 Class   c = Class.forName("org.netbeans.performance.test.utilities.LoggingScanClasspath",true,Thread.currentThread().getContextClassLoader()); // NOI18N
899                                 java.lang.reflect.Method   m = c.getMethod("reportScanOfFile", new Class  [] {String  .class, Long  .class}); // NOI18N
900                                 m.invoke(c.newInstance(), new Object  [] {rootURL.toExternalForm(), new Long  (endT - startT)});
901                             } catch (Exception   e) {
902                                     Exceptions.printStackTrace(e);
903                             }                            
904                         }
905                     }
906                 } catch (Throwable   e) {
907                     if (e instanceof ThreadDeath  ) {
908                         throw (ThreadDeath  ) e;
909                     }
910                     else {
911                         Exceptions.attachMessage(e, "While scanning: " + rootURL);
912                         Exceptions.printStackTrace(e);
913                     }
914                 }
915             }
916             for (java.util.ListIterator  <URL  > it = this.state.listIterator(this.state.size()); it.hasPrevious(); ) {
917                 if (this.canceled.getAndSet(false)) {                    
918                     return false;
919                 }
920                 try {
921                     final URL   rootURL = it.previous();
922                     it.remove();                                                                                
923                     if (!oldRoots.remove(rootURL)) {                        
924                         long startT = System.currentTimeMillis();
925                         updateFolder (rootURL,rootURL, false, handle);
926                         long endT = System.currentTimeMillis();
927                         if (PERF_TEST) {
928                             try {
929                                 Class   c = Class.forName("org.netbeans.performance.test.utilities.LoggingScanClasspath",true,Thread.currentThread().getContextClassLoader()); // NOI18N
930                                 java.lang.reflect.Method   m = c.getMethod("reportScanOfFile", new Class  [] {String  .class, Long  .class}); // NOI18N
931                                 m.invoke(c.newInstance(), new Object  [] {rootURL.toExternalForm(), new Long  (endT - startT)});
932                             } catch (Exception   e) {
933                                     Exceptions.printStackTrace(e);
934                             }
935                         }
936                         if (PREINDEXING) {
937                             // How do I obtain the data folder for this puppy?
938                             Index.preindex(rootURL);
939                         }
940                     }
941                 } catch (Throwable   e) {
942                     if (e instanceof ThreadDeath  ) {
943                         throw (ThreadDeath  ) e;
944                     }
945                     else {
946                         Exceptions.printStackTrace (e);
947                     }
948                 }
949             }
950             return true;
951         }
952         
953         private boolean isBoot(File   fo) {
954             // XXX Horrible hack!
955             //return fo.getPath().indexOf("jruby-0.9.8" + File.separatorChar + "lib" + File.separatorChar + "ruby" + File.separatorChar + "1.8") != -1; // XXX Hack
956             // XXX This is a hack but I should at least map 1.8 to RubyInstallation.RUBY_VERSION
957             return fo.getName().equals("1.8") || fo.getName().equals("rubystubs") || fo.getName().equals("ruby") || fo.getName().equals("lib");
958         }
959         
960         private void updateFolder(final URL   folder, final URL   root, boolean clean, final ProgressHandle handle) throws IOException   {
961             final FileObject rootFo = URLMapper.findFileObject(root);
962             if (rootFo == null) {
963                 return;
964             }            
965             if (!rootFo.isFolder()) {
966                 Logger.getLogger("global").warning("Source root has to be a folder: " +  FileUtil.getFileDisplayName(rootFo)); // NOI18N
967                 return;
968             }        
969             
970             final ClassPath sourcePath = ClassPath.getClassPath(rootFo,ClassPath.SOURCE);
971             final ClassPath bootPath = ClassPath.getClassPath(rootFo, ClassPath.BOOT);
972             final ClassPath compilePath = ClassPath.getClassPath(rootFo, ClassPath.COMPILE);            
973             final boolean isInitialCompilation = folder.equals(root);            
974             if (sourcePath == null || bootPath == null || compilePath == null) {
975                 Logger.getLogger("global").warning("Ignoring root with no ClassPath: " + FileUtil.getFileDisplayName(rootFo));    // NOI18N
976                 return;
977             }
978             boolean isBoot = isInitialCompilation && ClassIndexManager.getDefault().isBootRoot(root);
979             if (!clean && isInitialCompilation) {
980                 //Initial compilation  debug messages
981                 if (RepositoryUpdater.this.scannedRoots.contains(root)) {
982                     return;
983                 }                
984                 LOGGER.fine("Scanning Root: " + FileUtil.getFileDisplayName(rootFo));    //NOI18N
985             }
986             try {                                
987                 final File   rootFile = FileUtil.toFile(rootFo);
988                 if (rootFile == null) {
989                     // Probably a jar, which sometimes ends up in my updateFolder now because the
990                     // isBinary stuff etc. isn't working yet
991                     return;
992                 }
993                 final File   folderFile = isInitialCompilation ? rootFile : FileUtil.normalizeFile(new File   (URI.create(folder.toExternalForm())));
994                 if (handle != null) {
995                     final String   message = String.format (NbBundle.getMessage(RepositoryUpdater.class,"MSG_Scannig"),rootFile.getAbsolutePath());
996                     handle.setDisplayName(message);
997                 }
998 //                //Preprocessor support
999 //                JavaFileFilterImplementation filter = filters.get(root);
1000//                if (filter == null) {
1001//                    filter = JavaFileFilterQuery.getFilter(rootFo);
1002//                    if (filter != null) {
1003//                        if (!filters.values().contains(filter)) {
1004//                            filter.addChangeListener(filterListener);
1005//                        }
1006//                        filters.put(root, filter);
1007//                    }
1008//                }
1009                final ClasspathInfo cpInfo = ClasspathInfoAccessor.INSTANCE.create(CacheClassPath.forClassPath(bootPath),CacheClassPath.forClassPath(compilePath),sourcePath,null/*filter*/,true);
1010                List  <ParserFile> toCompile = new LinkedList  <ParserFile>();
1011                final File   classCache = Index.getClassFolder(rootFile);
1012                final Map   <String  ,List  <File  >> resources = getAllClassFiles(classCache, FileObjects.getRelativePath(rootFile,folderFile),true);
1013                final LazyFileList children = new LazyFileList(folderFile);
1014                ClassIndexImpl uqImpl = ClassIndexManager.getDefault().createUsagesQuery(root, true);
1015                assert uqImpl != null;
1016                SourceAnalyser sa = uqImpl.getSourceAnalyser();
1017                assert sa != null;
1018                
1019                // If this is a boot class path, don't update it... XXX
1020                if (isBoot(folderFile)) {
1021                    if (folderFile.exists() && folderFile.canRead()) {
1022                         if (sa.isUpToDate(null,folderFile.lastModified())) {
1023                             return;
1024                         }
1025                    }
1026                }
1027                
1028                Set  <File  > rs = new HashSet  <File  > ();
1029                boolean invalidIndex = isInitialCompilation && !sa.isValid();
1030                for (File   child : children) {       
1031                    if (invalidIndex || clean || dirtyCrossFiles.remove(child.toURI())) {
1032                        toCompile.add (FileObjects.fileFileObject(child, rootFile, isBoot, null/* filter*/));
1033                    }
1034                    else {
1035                        String   offset = FileObjects.getRelativePath(rootFile,child);
1036                        final int index = offset.lastIndexOf('.');  //NOI18N
1037                        if (index > -1) {
1038                            offset = offset.substring(0,index);
1039                        }
1040                        List  <File  > files = resources.remove(offset);
1041                        if  (files==null) {
1042                            toCompile.add(FileObjects.fileFileObject(child, rootFile, isBoot, null/*filter*/));
1043                        } else {
1044                            //boolean rsf = files.get(0).getName().endsWith(FileObjects.RS);
1045                            if (files.get(0).lastModified() < child.lastModified()) {
1046                                toCompile.add(FileObjects.fileFileObject(child, rootFile, isBoot, null/*filter*/));
1047                                for (File   toDelete : files) {                            
1048                                    toDelete.delete();
1049                                    //if (rsf) {                                    
1050                                    //    rsf = false;
1051                                    //}
1052                                    //else {
1053                                        String   className = FileObjects.getBinaryName(toDelete,classCache);
1054                                        sa.delete(className);
1055                                    //}                            
1056                                }
1057                            }
1058                            //else if (rsf) {
1059                            //    files.remove(0);
1060                            //    rs.addAll(files);
1061                            //}                        
1062                        }
1063                    }
1064                }
1065                for (List  <File  > files : resources.values()) {
1066                    for (File   toDelete : files) {
1067                        if (!rs.contains(toDelete)) {
1068                            toDelete.delete();
1069                            if (toDelete.getName().endsWith(FileObjects.SIG)) {
1070                                String   className = FileObjects.getBinaryName(toDelete,classCache);                        
1071                                sa.delete(className);
1072                            }
1073                        }
1074                    }
1075                }
1076                if (!toCompile.isEmpty()) {
1077                    if (handle != null) {
1078                        // BEGIN TOR MODIFICATIONS
1079                        // Show message for "indexing" rather than compiling since I'm not keeping trees around etc - it's
1080                        // all used to populate Lucene at this point.
1081                        //final String message = String.format (NbBundle.getMessage(RepositoryUpdater.class,"MSG_BackgroundCompile"),rootFile.getAbsolutePath());
1082                        String   path = rootFile.getAbsolutePath();
1083                        // Shorten path by prefix to ruby location if possible
1084                        int rubyIndex = path.indexOf("jruby-0.9.8");
1085                        if (rubyIndex != -1) {
1086                            path = path.substring(rubyIndex);
1087                        }
1088                        final String   message = String.format (NbBundle.getMessage(RepositoryUpdater.class,"MSG_Analyzing"),path);
1089                        handle.setDisplayName(message);
1090                    }
1091                    batchCompile(toCompile, rootFo, cpInfo, sa, dirtyCrossFiles, handle);                    
1092                }
1093                sa.store();
1094            } finally {
1095                if (!clean && isInitialCompilation) {
1096                    RepositoryUpdater.this.scannedRoots.add(root);
1097                }
1098            }
1099        }
1100        
1101        private void updateFile (final URL   file, final URL   root) throws IOException   {
1102            final FileObject fo = URLMapper.findFileObject(file);
1103            if (fo == null) {
1104                return;
1105            }
1106
1107            Language language = LanguageRegistry.getInstance().getLanguageByMimeType(fo.getMIMEType());
1108            if (language == null) {
1109                return;
1110            }
1111            
1112            assert "file".equals(root.getProtocol()) : "Unexpected protocol of URL: " + root;   //NOI18N
1113            final ClassIndexImpl uqImpl = ClassIndexManager.getDefault().createUsagesQuery(root, true);
1114            if (uqImpl != null) {
1115                uqImpl.setDirty(null);
1116//                final JavaFileFilterImplementation filter = JavaFileFilterQuery.getFilter(fo);
1117                ClasspathInfo cpInfo = ClasspathInfoAccessor.INSTANCE.create (fo, null/*filter*/, true);
1118                final File   rootFile = FileUtil.normalizeFile(new File   (URI.create(root.toExternalForm())));
1119                final File   fileFile = FileUtil.toFile(fo);
1120                //final File classCache = Index.getClassFolder (rootFile);
1121                //final Map <String,List<File>> resources = getAllClassFiles (classCache, /*FileObjects.*/getRelativePath(rootFile, fileFile.getParentFile()),false);
1122//                String offset = FileObjects.getRelativePath (rootFile,fileFile);
1123//                final int index = offset.lastIndexOf('.');  //NOI18N
1124//                if (index > -1) {
1125//                    offset = offset.substring(0,index);
1126//                }
1127                //List<File> files = resources.remove (offset);                
1128                
1129                SourceAnalyser sa = uqImpl.getSourceAnalyser();
1130                
1131                // TODO - handle deletions
1132                assert sa != null;
1133//                if (files != null) {
1134//                    for (File toDelete : files) {
1135//                        toDelete.delete();
1136//                        if (toDelete.getName().endsWith(FileObjects.SIG)) {
1137//                            String className = FileObjects.getBinaryName (toDelete,classCache);                                                           
1138//                            sa.delete (className);
1139//                        }
1140//                    }
1141//                }
1142//                else {
1143//                    sa.delete(FileObjects.convertFolder2Package(offset, '/'));  //NOI18N
1144//                }
1145                assert fo != null;
1146                String   sourceLevel = SourceLevelQuery.getSourceLevel(fo);
1147                final CompilerListener listener = new CompilerListener ();
1148                //final JavaFileManager fm = ClasspathInfoAccessor.INSTANCE.getFileManager(cpInfo);                
1149                ParserFile active = FileObjects.fileFileObject(fileFile, rootFile, false, null/*filter*/);
1150                
1151                ParserTaskImpl jt = SourceAccessor.INSTANCE.createParserTask(language, cpInfo, sourceLevel);
1152                jt.setParseListener(listener);
1153                Iterable  <ParserResult> trees = jt.parse(new ParserFile[] { active });
1154                sa.analyse (trees, jt, /*fm,*/ active);
1155                listener.cleanDiagnostics();
1156                sa.store();
1157            }
1158        }
1159        
1160        private void delete (final URL   file, final URL   root, final boolean folder) throws IOException   {
1161            assert "file".equals(root.getProtocol()) : "Unexpected protocol of URL: " + root;   //NOI18N
1162            final File   rootFile = FileUtil.normalizeFile(new File   (URI.create(root.toExternalForm())));
1163            assert "file".equals(file.getProtocol()) : "Unexpected protocol of URL: " + file;   //NOI18N
1164            final File   fileFile = FileUtil.normalizeFile(new File   (URI.create(file.toExternalForm())));
1165            final String   offset = FileObjects.getRelativePath (rootFile,fileFile);
1166            assert offset != null && offset.length() > 0 : String.format("File %s not under root %s ", fileFile.getAbsolutePath(), rootFile.getAbsolutePath());  // NOI18N                        
1167            final File   classCache = Index.getClassFolder (rootFile);
1168            File  [] affectedFiles = null;
1169            if (folder) {
1170                final File   container = new File   (classCache, offset);
1171                affectedFiles = container.listFiles();
1172            }
1173            else {
1174                int slashIndex = offset.lastIndexOf (File.separatorChar);
1175                int dotIndex = offset.lastIndexOf('.');     //NOI18N
1176                final File   container = slashIndex == -1 ? classCache : new File   (classCache,offset.substring(0,slashIndex));
1177                final String   name = offset.substring(slashIndex+1, dotIndex);
1178                final String  [] patterns = new String  [] {
1179                  name + '.',
1180                  name + '$'
1181                };
1182                final File  [] content  = container.listFiles();
1183                if (content != null) {
1184                    final List  <File  > result = new ArrayList  <File  >(content.length);
1185                    for (File   f : content) {
1186                        final String   fname = f.getName();
1187                        if (fname.startsWith(patterns[0]) || fname.startsWith(patterns[1])) {
1188                            result.add(f);
1189                        }
1190                    }
1191                    affectedFiles = result.toArray(new File  [result.size()]);
1192                }
1193            }
1194            if (affectedFiles != null && affectedFiles.length > 0) {
1195                final ClassIndexImpl uqImpl = ClassIndexManager.getDefault().createUsagesQuery(root, true);
1196                assert uqImpl != null;                
1197                final SourceAnalyser sa = uqImpl.getSourceAnalyser();
1198                assert sa != null;
1199                for (File   f : affectedFiles) {
1200                    if (f.getName().endsWith(FileObjects.RS)) {
1201                        List  <File  > rsFiles = new LinkedList  <File  >();
1202                        readRSFile(f, classCache, rsFiles);
1203                        for (File   rsf : rsFiles) {
1204                            String   className = FileObjects.getBinaryName (rsf,classCache);                                                                        
1205                            sa.delete (className);
1206                            rsf.delete();
1207                        }
1208                    }
1209                    else {
1210                        String   className = FileObjects.getBinaryName (f,classCache);                                                                        
1211                        sa.delete (className);
1212                    }
1213                    f.delete();                    
1214                }
1215                sa.store();
1216            }
1217        }
1218        
1219        private void updateBinary (final URL   file, final URL   root) throws IOException   {
1220            //CachingArchiveProvider.getDefault().clearArchive(root);  
1221            File   rootFile = FileObjects.getRootFile(root);
1222            if (rootFile.exists()) {
1223                final BinaryAnalyser ba = ClassIndexManager.getDefault().createUsagesQuery(root, false).getBinaryAnalyser();
1224                ba.analyse(rootFile, handle);
1225            }
1226        }                
1227    }        
1228    
1229    static class LazyFileList implements Iterable  <File  > {
1230    
1231        private File   root;
1232
1233        public LazyFileList (final File   root) {
1234            assert root != null;
1235            this.root = root;
1236        }
1237
1238        public Iterator  <File  > iterator() {
1239            if (!root.exists()) {
1240                return Collections.<File  >emptySet().iterator();
1241            }
1242            return new It (this.root);
1243        }
1244
1245
1246        private class It implements Iterator  <File  > {
1247
1248            private final List  <File  > toDo = new LinkedList  <File  > ();
1249
1250            public It (File   root) {
1251                this.toDo.addAll (java.util.Arrays.asList(root.listFiles()));
1252            }
1253
1254            public boolean hasNext() {
1255                while (!toDo.isEmpty()) {
1256                    File   f = toDo.remove (0);   
1257                    final String   name = f.getName();
1258                    if (f.isDirectory() && !ignoredDirectories.contains(name)/* && Utilities.isJavaIdentifier(name)*/) {
1259                        File  [] content = f.listFiles();
1260                            for (int i=0,j=0;i<content.length;i++) {
1261                                f = content[i];
1262                                if (f.isFile()) {
1263                                    this.toDo.add(j++,f);
1264                                }
1265                                else {
1266                                    this.toDo.add(f);
1267                                }
1268                            }
1269                    }                    
1270                    else { // XXX How do I decide if it's a reasonable name?
1271//                        System.out.println("Should we scan " + name + "?");
1272//                    else if (name.endsWith('.'+JavaDataLoader.JAVA_EXTENSION) && !PACKAGE_INFO.equals(name) && f.length()>0) { //NOI18N
1273                        toDo.add(0,f);
1274                        return true;
1275                    }                                        
1276                }
1277                return false;
1278            }
1279
1280            public File   next() {
1281                return toDo.remove (0);
1282            }
1283
1284            public void remove() {
1285                throw new UnsupportedOperationException   ();
1286            }
1287
1288        }            
1289    }
1290    
1291    
1292    private final class Delay {
1293        
1294        private final Timer   timer;
1295        private final List  <Work> tasks;
1296        
1297        public Delay () {
1298            this.timer = new Timer   ();
1299            this.tasks = new LinkedList  <Work> ();
1300        }
1301        
1302        public synchronized void post (final Work work) {
1303            assert work != null;
1304            this.tasks.add (work);
1305            this.timer.schedule(new DelayTask (work),DELAY);
1306        }
1307        
1308        public void cancel () {
1309            Work[] toCancel;
1310            synchronized (this) {
1311                toCancel = this.tasks.toArray (new Work[this.tasks.size()]);
1312            }
1313            for (Work w : toCancel) {
1314                if (w.workType == WorkType.COMPILE) {
1315                    w = new SingleRootWork (WorkType.DELETE,((SingleRootWork)w).file,
1316                        ((SingleRootWork)w).root,((SingleRootWork)w).isFolder,
1317                        w.latch);
1318                }
1319                CompileWorker cw = new CompileWorker (w);
1320                try {
1321                    cw.run (null);
1322                } catch (IOException   ioe) {
1323                    Exceptions.printStackTrace(ioe);
1324                }
1325            }
1326        }
1327        
1328        
1329        private class DelayTask extends TimerTask   {
1330            
1331            final Work work;
1332            
1333            public DelayTask (final Work work) {
1334                this.work = work;
1335            }
1336            
1337            public void run() {
1338                submit(work);
1339                synchronized (Delay.this) {
1340                    Delay.this.tasks.remove (work);
1341                }                
1342            }
1343
1344            public @Override   boolean cancel() {
1345                boolean retValue = super.cancel();
1346                if (retValue) {
1347                    synchronized (Delay.this) {
1348                        Delay.this.tasks.remove (work);
1349                    }
1350                }
1351                return retValue;
1352            }                        
1353        }                
1354    }
1355    
1356    private static class CompilerListener implements /*DiagnosticListener<JavaFileObject>,*/LowMemoryListener, ParseListener {
1357                               
1358        final List  <Error  > errors = new LinkedList  <Error  > ();
1359        final List  <Error  > warnings = new LinkedList  <Error  > ();
1360//        final List<ClassSymbol> justEntered = new LinkedList<ClassSymbol> ();
1361        final List  <ParserResult> justEntered = new LinkedList  <ParserResult> ();
1362        final AtomicBoolean   lowMemory = new AtomicBoolean   ();
1363//        
1364        void cleanDiagnostics () {
1365            if (!this.errors.isEmpty()) {
1366                if (LOGGER.isLoggable(Level.FINE)) {
1367                    for (Error   msg : this.errors) {
1368                        LOGGER.fine(msg.toString());      //NOI18N
1369                    }
1370                }
1371                this.errors.clear();
1372            }
1373            if (!this.warnings.isEmpty()) {
1374                if (LOGGER.isLoggable(Level.FINE)) {
1375                    for (Error   msg: this.warnings) {
1376                        LOGGER.fine(msg.toString());      //NOI18N
1377                    }
1378                }
1379                this.warnings.clear();
1380            }
1381            this.justEntered.clear();
1382        }
1383        
1384        List  <ParserResult> getEnteredTypes () {
1385            List  <ParserResult> result = new ArrayList  <ParserResult>(this.justEntered);
1386            this.justEntered.clear();
1387            return result;
1388        }
1389        
1390        public void error(Error   error) {
1391            if (error.getSeverity() == Severity.ERROR) {
1392                this.errors.add(error);
1393            } else {
1394                this.warnings.add(error);
1395            }
1396        }
1397        
1398        public void exception(Exception   exception) {
1399        }
1400        
1401//
1402//        public void report(final Diagnostic diagnosticMessage) {
1403//            Diagnostic.Kind kind = diagnosticMessage.getKind();
1404//            if ( kind == Diagnostic.Kind.ERROR) {
1405//                this.errors.add (diagnosticMessage);
1406//            }
1407//            else if (kind == Diagnostic.Kind.WARNING
1408//                   ||kind == Diagnostic.Kind.MANDATORY_WARNING) {
1409//                this.warnings.add (diagnosticMessage);
1410//            }
1411//        }
1412//        
1413        public void started(ParseEvent e) {
1414            
1415        }
1416
1417        public void finished(ParseEvent event) {
1418            if (event.getKind() == ParseEvent.Kind.PARSE /*ENTER*/) {
1419                //final CompilationUnitTree unit = event.getCompilationUnit();
1420                final ParserResult result = event.getResult();
1421                if (result != null) {
1422                    this.justEntered.add(result);
1423                }
1424//                
1425//                for (Tree typeTree : unit.getTypeDecls()) {
1426//                    if (typeTree instanceof JCTree.JCClassDecl) {       //May be a JCTree.JCExpressionStatement in case of an error
1427//                        ClassSymbol sym = ((JCTree.JCClassDecl)typeTree).sym;
1428//                        if (sym != null) {
1429//                            if (sym.sourcefile == null) {
1430//                                sym.sourcefile = event.getSourceFile();
1431//                            }
1432//                            this.justEntered.add(sym);
1433//                        }
1434//                    }          
1435//                }
1436            }
1437        }
1438        
1439        public void lowMemory (final LowMemoryEvent event) {
1440            this.lowMemory.set(true);
1441        }
1442    }
1443    
1444    public static void batchCompile (final List  <ParserFile> toCompile, final FileObject rootFo, final ClasspathInfo cpInfo, final SourceAnalyser sa, final Set  <URI  > dirtyFiles, ProgressHandle handle) throws IOException   {
1445        assert toCompile != null;
1446        assert rootFo != null;
1447        assert cpInfo != null;
1448        ParserFile active = null;
1449        //final JavaFileFilterImplementation filter = JavaFileFilterQuery.getFilter(rootFo);
1450        Object   filter;
1451        //final JavaFileManager fileManager = ClasspathInfoAccessor.INSTANCE.getFileManager(cpInfo);
1452        final CompilerListener listener = new CompilerListener ();        
1453        LowMemoryNotifier.getDefault().addLowMemoryListener(listener);
1454        try {
1455            ParserTaskImpl jt = null;
1456            
1457            try {
1458                List  <ParserFile> bigFiles = new LinkedList  <ParserFile>();
1459                int state = 0; // TODO: Document what these states mean
1460                boolean isBigFile = false;
1461                final String   sourceLevel = SourceLevelQuery.getSourceLevel(rootFo);
1462                int fileNumber = 0;
1463                int fileCount = toCompile.size();
1464                if (fileCount > 0) {
1465                    handle.switchToDeterminate(fileCount);
1466                }
1467fileIter:                    
1468                while (!toCompile.isEmpty() || !bigFiles.isEmpty() || active != null) {
1469                    try {
1470                        if (listener.lowMemory.getAndSet(false)) {
1471                            if (jt != null) {
1472                                jt.finish();
1473                            }
1474                            jt = null;
1475                            if (state == 1) {
1476                                break;
1477                            } else {
1478                                state = 1;
1479                            }
1480                            System.gc();
1481                            continue;
1482                        }
1483                        if (active == null) {
1484                            if (!toCompile.isEmpty()) {
1485                                active = toCompile.remove(0);
1486                                isBigFile = false;
1487                            } else {
1488                                active = bigFiles.remove(0);
1489                                isBigFile = true;
1490                            }
1491                        } 
1492                        if (handle != null && active != null) {
1493                            // BEGIN TOR MODIFICATIONS
1494                            //final String message = String.format (NbBundle.getMessage(RepositoryUpdater.class,"MSG_Analyzing"),active.getNameExt());
1495                            //handle.setDisplayName(message);
1496                            if (fileCount > 0 && fileNumber <= fileCount) {
1497                                handle.progress(fileNumber);
1498                            }
1499                            fileNumber++;
1500                        }
1501                        
1502                        boolean compiled = false;
1503                        
1504                    // Change from what's going on in the Retouche updater:
1505                    // We could have many language implementations that want to index this root;
1506                    // we need to iterate through them and let each one of them index if they
1507                    // want to.    
1508                    List  <IndexerEntry> indexers = getIndexers();
1509                    assert indexers instanceof RandomAccess  ;
1510                    // We're gonna do this for every file in the filesystem - do cheaper iteration
1511                    // using indices rather than iterators
1512                    //for (IndexerEntry entry : getIndexers()) {
1513                    for (int in = 0; in < indexers.size(); in++) {
1514                        IndexerEntry entry = indexers.get(in);
1515                        Language language = entry.getLanguage();
1516                        Indexer indexer = entry.getIndexer();
1517                        if (!indexer.isIndexable(active)) {
1518                            continue;
1519                        }
1520                        compiled = true;
1521                        
1522                        // Cache parser tasks per indexer
1523                        jt = entry.getParserTask();
1524                
1525                        if (jt == null) {
1526                            jt = SourceAccessor.INSTANCE.createParserTask(language, cpInfo/*, listener*/, sourceLevel);
1527                            jt.setParseListener(listener);
1528                            entry.setParserTask(jt);
1529                            LOGGER.fine("Created new ParserTask for: " + FileUtil.getFileDisplayName(rootFo));    //NOI18N
1530                        }
1531                        Iterable  <ParserResult> trees = jt.parse(new ParserFile[] { active });
1532                        if (listener.lowMemory.getAndSet(false)) {
1533                            jt.finish();
1534                            jt = null;
1535                            listener.cleanDiagnostics();
1536                            trees = null;
1537                            if (state == 1) {
1538                                if (isBigFile) {
1539                                    break fileIter;
1540                                } else {
1541                                    bigFiles.add(active);
1542                                    active = null;
1543                                    state = 0;
1544                                }
1545                            } else {
1546                                state = 1;
1547                            }
1548                            System.gc();
1549                            continue fileIter;
1550                        }
1551                        //Iterable<? extends TypeElement> types = jt.enterTrees(trees);
1552//                        dumpClasses (listener.getEnteredTypes()/*,fileManager, 
1553//                                com.sun.tools.javac.code.Types.instance(jt.getContext()),
1554//                                com.sun.tools.javac.util.Name.Table.instance(jt.getContext())*/);
1555//                        if (listener.lowMemory.getAndSet(false)) {
1556//                            jt.finish();
1557//                            jt = null;
1558//                            listener.cleanDiagnostics();
1559//                            trees = null;
1560//                            //types = null;
1561//                            if (state == 1) {
1562//                                if (isBigFile) {
1563//                                    break fileIter;
1564//                                } else {
1565//                                    bigFiles.add(active);
1566//                                    active = null;
1567//                                    state = 0;
1568//                                }
1569//                            } else {
1570//                                state = 1;
1571//                            }
1572//                            System.gc();
1573//                            continue fileIter;
1574//                        }                        
1575//                        final JavaCompiler jc = JavaCompiler.instance(jt.getContext());
1576//                        final /*Java*/FileObject finalActive = active;
1577//                        Filter f = new Filter() {
1578//                            public void process(Env<AttrContext> env) {
1579//                                try {
1580//                                    jc.attribute(env);
1581//                                } catch (Throwable t) {
1582//                                    if (finalActive.toUri().getPath().contains("org/openide/loaders/OpenSupport.java")) {
1583//                                        Exceptions.printStackTrace(t);
1584//                                    }
1585//                                }
1586//                            }
1587//                        };
1588//                        f.run(jc.todo, types);
1589//                        dumpClasses (listener.getEnteredTypes()/*, fileManager,
1590//                                com.sun.tools.javac.code.Types.instance(jt.getContext()),
1591//                                com.sun.tools.javac.util.Name.Table.instance(jt.getContext())*/);
1592//                        if (listener.lowMemory.getAndSet(false)) {
1593//                            jt.finish();
1594//                            jt = null;
1595//                            listener.cleanDiagnostics();
1596//                            trees = null;
1597//                            //types = null;
1598//                            if (state == 1) {
1599//                                if (isBigFile) {
1600//                                    break fileIter;
1601//                                } else {
1602//                                    bigFiles.add(active);
1603//                                    active = null;
1604//                                    state = 0;
1605//                                }
1606//                            } else {
1607//                                state = 1;
1608//                            }
1609//                            System.gc();
1610//                            continue fileIter;
1611//                        }
1612                        if (sa != null && trees != null) {
1613                            sa.analyse(trees, jt,/* ClasspathInfoAccessor.INSTANCE.getFileManager(cpInfo),*/ active);
1614                        }                                        
1615                        if (!listener.errors.isEmpty()) {
1616                            //Log.instance(jt.getContext()).nerrors = 0;
1617                            listener.cleanDiagnostics();
1618                        }
1619                        active = null;
1620                        state  = 0;
1621                    } // End of iteration through indexers for this file (active)
1622                    if (!compiled) {
1623                        // No indexers were interested in this file - clean it out
1624                        active = null;
1625                        listener.cleanDiagnostics();
1626                        continue;
1627                    }
1628
1629                    } catch (Throwable   t) {
1630                        if (t instanceof ThreadDeath  ) {
1631                            throw (ThreadDeath  ) t;
1632                        }
1633                        else {
1634                            if (jt != null) {
1635                                jt.finish();
1636                            }
1637                            String   activeURI;
1638                            if (active != null) {
1639                                activeURI = active.getNameExt();
1640                            } else {
1641                                activeURI = "unknown";
1642                            }
1643                            jt = null;
1644                            active = null;                            
1645                            listener.cleanDiagnostics();
1646                            //if (!(t instanceof Abort)) { // a javac Throwable                                
1647                                final ClassPath bootPath   = cpInfo.getClassPath(ClasspathInfo.PathKind.BOOT);
1648                                final ClassPath classPath  = cpInfo.getClassPath(ClasspathInfo.PathKind.COMPILE);
1649                                final ClassPath sourcePath = cpInfo.getClassPath(ClasspathInfo.PathKind.SOURCE);
1650                                t = Exceptions.attachMessage(t,String.format("Root: %s File: %s Bootpath: %s Classpath: %s Sourcepath: %s",
1651                                        FileUtil.getFileDisplayName(rootFo),
1652                                        activeURI.toString(),
1653                                        bootPath == null   ? null : bootPath.toString(),
1654                                        classPath == null  ? null : classPath.toString(),
1655                                        sourcePath == null ? null : sourcePath.toString()
1656                                        ));
1657                                Exceptions.printStackTrace(t);
1658                            //}
1659                        }
1660                    }
1661                }
1662                if (state == 1) {
1663                    Logger.getLogger("global").warning("Not enough memory to compile folder: " + FileUtil.getFileDisplayName(rootFo));    // NOI18N
1664                }
1665            } finally {
1666                if (jt != null) {
1667                    jt.finish();
1668                }
1669            }
1670        } finally {
1671            clearIndexerParserTasks();
1672
1673            LowMemoryNotifier.getDefault().removeLowMemoryListener(listener);
1674        }
1675    }
1676    
1677    private static Set  <String  > parseSet(String   propertyName, String   defaultValue) {
1678        StringTokenizer   st = new StringTokenizer  (System.getProperty(propertyName, defaultValue), " \t\n\r\f,-:+!");
1679        Set  <String  > result = new HashSet  <String  >();
1680        while (st.hasMoreTokens()) {
1681            result.add(st.nextToken());
1682        }
1683        return result;
1684    }
1685    
1686    
1687    public static Map  <String  ,List  <File  >> getAllClassFiles (final File   root, final String   offset, boolean recursive) {
1688        assert root != null;
1689        Map  <String  ,List  <File  >> result = new HashMap  <String  ,List  <File  >> ();
1690        String   rootName = root.getAbsolutePath();
1691        int len = rootName.length();
1692        if (rootName.charAt(len-1)!=File.separatorChar) {
1693            len++;
1694        }
1695        File   folder = root;
1696        if (offset.length() > 0) {
1697            folder = new File   (folder,offset);  //NOI18N
1698            if (!folder.exists() || !folder.isDirectory()) {
1699                return result;
1700            }
1701        }
1702        getAllClassFilesImpl (folder, root,len,result, recursive);
1703        return result;
1704    }
1705        
1706    private static void getAllClassFilesImpl (final File   folder, final File   root, final int oi, final Map  <String  ,List  <File  >> result, final boolean recursive) {
1707        final File  [] content = folder.listFiles();
1708        for (File   f: content) {
1709            if (f.isDirectory() && recursive) {
1710                getAllClassFilesImpl(f, root, oi,result, recursive);                    
1711            }
1712            else {
1713                String   path = f.getAbsolutePath();
1714                int extIndex = path.lastIndexOf('.');  //NO18N
1715                if (extIndex+1+FileObjects.RS.length() == path.length() && path.endsWith(FileObjects.RS)) {
1716                    path = path.substring (oi,extIndex);
1717                    List  <File  > files = result.get (path);
1718                    if (files == null) {
1719                        files = new LinkedList  <File  >();
1720                        result.put (path,files);
1721                    }
1722                    files.add(0,f); //the rs file has to be the first
1723                    try {
1724                        readRSFile (f,root, files);
1725                    } catch (IOException   ioe) {
1726                        //The signature file is broken, report it but don't stop scanning
1727                        Exceptions.printStackTrace(ioe);
1728                    }
1729                }
1730                else if (extIndex+1+FileObjects.SIG.length() == path.length() && path.endsWith(FileObjects.SIG)) {
1731                    int index = path.indexOf('$',oi);  //NOI18N                    
1732                    if (index == -1) {
1733                        path = path.substring (oi,extIndex);
1734                    }
1735                    else {
1736                        path = path.substring (oi,index);
1737                    }                    
1738                    List  <File  > files = result.get (path);
1739                    if (files == null) {
1740                        files = new LinkedList  <File  >();
1741                        result.put (path,files);
1742                    }
1743                    files.add (f);
1744                }
1745            }
1746        }
1747    }
1748        
1749    private static void readRSFile (final File   f, final File   root, final List  <? super File  > files) throws IOException   {
1750        BufferedReader   in = new BufferedReader   (new FileReader   (f));
1751        try {
1752            String   binaryName;
1753            while ((binaryName=in.readLine())!=null) {
1754                File   sf = new File   (root, FileObjects.convertPackage2Folder(binaryName)+'.'+FileObjects.SIG);
1755                files.add(sf);                                        
1756            }
1757        } finally {
1758            in.close();
1759        }
1760    }
1761    
1762    
1763    private class FilterListener implements ChangeListener   {
1764            
1765        public void stateChanged(ChangeEvent   event) {
1766//            Object source = event.getSource();
1767//            System.out.println("ignoring statechange " + event + " since I have no filter");
1768//            if (source instanceof JavaFileFilterImplementation) {
1769//                List<URL> dirtyRoots = new LinkedList<URL> ();
1770//                synchronized (filters) {
1771//                    for (Map.Entry<URL,JavaFileFilterImplementation> e : filters.entrySet()) {
1772//                        if (e.getValue() == source) {
1773//                            dirtyRoots.add(e.getKey());
1774//                        }
1775//                    }
1776//                }
1777//                submit(Work.filterChange(dirtyRoots));
1778//            }
1779        }
1780    }
1781    
1782    public static synchronized RepositoryUpdater getDefault () {
1783        if (instance == null) {
1784            instance = new RepositoryUpdater ();
1785        }
1786        return instance;
1787    }        
1788  
1789    // BEGIN TOR MODIFICATIONS
1790    // There could be multiple indexers (for different languages) that want
1791    // to index a given file. I will iterate over the indexers and let each
1792    // indexer have a chance to index every file. To do this I compute
1793    // a list of indexers in advance - and provide a place where we can
1794    // cache the parser tasks such that only one is created per indexer.
1795    
1796    private static List  <IndexerEntry> indexers;
1797    
1798    private static List  <IndexerEntry> getIndexers() {
1799        if (indexers == null) {
1800            indexers = new ArrayList  <IndexerEntry>();
1801            for (Language language : LanguageRegistry.getInstance()) {
1802                Indexer indexer = language.getIndexer();
1803                if (indexer != null) {
1804                    IndexerEntry entry = new IndexerEntry(language, indexer);
1805                    indexers.add(entry);
1806                }
1807            }
1808        }
1809        
1810        return indexers;
1811    }
1812    
1813    private static void clearIndexerParserTasks() {
1814        for (IndexerEntry entry : getIndexers()) {
1815            entry.setParserTask(null);
1816        }
1817    }
1818    
1819    private static class IndexerEntry {
1820        private Language language;
1821        private Indexer indexer;
1822        private ParserTaskImpl task;
1823        
1824        IndexerEntry(Language language, Indexer indexer) {
1825            this.language = language;
1826            this.indexer = indexer;
1827        }
1828        
1829        Indexer getIndexer() {
1830            return indexer;
1831        }
1832        
1833        Language getLanguage() {
1834            return language;
1835        }
1836        
1837        ParserTaskImpl getParserTask() {
1838            return task;
1839        }
1840        
1841        void setParserTask(ParserTaskImpl task) {
1842            this.task = task;
1843        }
1844    } 
1845    
1846    // END TOR MODIFICATIONS
1847}
1848
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags