Java > Open Source Codes > de > gulden > util > javasource > SourceParser


1 /*
2  * Project: BeautyJ - Customizable Java Source Code Transformer
3  * Class: de.gulden.util.javasource.SourceParser
4  * Version: 1.1
5  *
6  * Date: 2004-09-29
7  *
8  * Note: Contains auto-generated Javadoc comments created by BeautyJ.
9  *
10  * This is licensed under the GNU General Public License (GPL)
11  * and comes with NO WARRANTY. See file license.txt for details.
12  *
13  * Author: Jens Gulden
14  * Email: beautyj@jensgulden.de
15  */
16
17 package de.gulden.util.javasource;
18
19 import de.gulden.util.javasource.sourclet.Sourclet;
20 import de.gulden.util.javasource.jjt.Node;
21 import de.gulden.util.javasource.jjt.*;
22 import javax.xml.parsers.*;
23 import org.w3c.dom.*;
24 import org.xml.sax.SAXException ;
25 import java.io.*;
26 import java.util.*;
27
28 /**
29  * SourceParser main utitility class.
30  * This class provides functionality to parse Java source codes and build
31  * a tree of objects representing the source elements.
32  * This tree of objects can be output as XML or, instead of parsing Java
33  * sources, be parsed from previously generated XML.
34  *
35  * @author Jens Gulden
36  * @version 1.1
37  * @see de.gulden.util.javasource.sourclet.Sourclet
38  * @see de.gulden.util.javasource.sourclet.standard.StandardSourclet
39  */
40 public class SourceParser implements ParserTreeConstants {
41
42     // ------------------------------------------------------------------------
43 // --- final static fields ---
44 // ------------------------------------------------------------------------
45
46     /**
47      * Version
48      */
49     public static final String VERSION = "1.1";
50
51     /**
52      * Constant workaroundUnicodeSingleCharMarker.
53      */
54     protected static final String workaroundUnicodeSingleCharMarker = "-" + "unicodechar" + "-";
55
56
57     // ------------------------------------------------------------------------
58 // --- static fields ---
59 // ------------------------------------------------------------------------
60
61     /**
62      * Linefeed.
63      */
64     public static String nl = System.getProperty("line.separator");
65
66     /**
67      * Flag specifying whether to include a DTD reference (<!DOCTYPE..>) into generated XML.
68      * Externally set.
69      */
70     public static boolean includeXMLDoctype = false;
71
72     /**
73      * Flag specifying whether to validate an XML file against its DTD before it is parsed.
74      * Externally set.
75      */
76     public static boolean validateXML = false;
77
78     /**
79      * Global verbose flag.
80      */
81     public static boolean verbose = false;
82
83     /**
84      * Log performer, may be set externally.
85      */
86     public static LogPerformer logPerformer = LogPerformer.DEFAULT;
87
88     /**
89      * Document builder for parsing XML.
90      * Will be initialized when first used.
91      */
92     protected static DocumentBuilder documentBuilder;
93
94
95     // ------------------------------------------------------------------------
96 // --- static methods ---
97 // ------------------------------------------------------------------------
98
99     /**
100      * Create object tree from Java source inputs.
101      *
102      * @throws IOException if an i/o error occurs
103      * @return Root package (named "") containing all other packages with classes.
104      */
105     public static Package parse(File file, ProgressTracker pt) throws IOException, ParseException {
106         return parse(new File[] {file}, pt);
107     }
108
109     /**
110      * Create object tree from Java source inputs.
111      *
112      * @param files A list of files and/or directories. Any .java-file will be parsed, any other ignored.
113      * @throws IOException if an i/o error occurs
114      * @return Root package (named "") containing all other packages with classes.
115      */
116     public static Package parse(File[] files, ProgressTracker pt) throws IOException, ParseException {
117         Package base=new Package ();
118         parse(files, base, pt);
119         return base;
120     }
121
122     /**
123      * Parses files and adds the parsed objects to the specified base package.
124      *
125      * @throws IOException if an i/o error occurs
126      */
127     public static void parse(File[] files, Package basePackage, ProgressTracker pt) throws IOException, ParseException {
128         parsePass1(basePackage ,files,pt);
129         analysePass2(basePackage ,pt);
130     }
131
132     /**
133      * Parses a file and adds the parsed objects to the specified bas package.
134      *
135      * @throws IOException if an i/o error occurs
136      */
137     public static void parse(File file, Package basePackage, ProgressTracker pt) throws IOException, ParseException {
138         parse(new File[] {file},basePackage, pt);
139     }
140
141     /**
142      * Create object tree from Java source inputs.
143      *
144      * @throws IOException if an i/o error occurs
145      * @return Base package (named "") containing all other packages with classes.
146      */
147     public static Package parse(String [] filenames, ProgressTracker pt) throws IOException, ParseException {
148         File[] f=new File[filenames.length];
149         for (int i=0;i<filenames.length;i++) {
150             f[i]=new File(filenames[i]);
151         }
152         return parse(f,pt);
153     }
154
155     /**
156      * Create object tree from XML input, previously created from parsed .java-files.
157      *
158      * @throws IOException if an i/o error occurs
159      * @throws SAXException if an XML parser error occurs
160      * @return Base package (named "") containing all other packages with classes.
161      * @see #buildXML
162      */
163     public static Package parseXML(InputStream in) throws IOException, SAXException {
164         Document doc=getDocumentBuilder().parse(in);
165         Package p=new Package ();
166         Element e=doc.getDocumentElement();
167         p.initFromXML(e); // <xjava> may be treated as base package
168 return p;
169     }
170
171     /**
172      * Output an object tree of source code elements to XML.
173      *
174      * @param p Package containing all other packages with classes that are to be converted to XML.
175      * @return The XML DOM-document.
176      * @see #parseXML
177      */
178     public static Document buildXML(Package p) {
179         DOMImplementation domImplementation=getDocumentBuilder().getDOMImplementation();
180         DocumentType doctype;
181         if (includeXMLDoctype) {
182             doctype=domImplementation.createDocumentType("xjava",null,"xjava.dtd");
183         } else {
184             doctype=null;
185         }
186         Document doc=domImplementation.createDocument(null,"xjava",doctype);
187         Element root=doc.getDocumentElement();
188         Element xml=p.buildXML(doc);
189         if (root!=xml) { // xml may be tag 'xjava', created by base package
190 root.appendChild(xml);
191         }
192         root.setAttribute("version", VERSION);
193         return doc;
194     }
195
196     /**
197      * Output object tree of source code elements as Java source files,
198      * applying a Sourclet for formatting the code.
199      *
200      * @param p Package containing all other packages with classes that are to be output as formatted source code.
201      * @param dir Base directory where to output .java-files. A directory structure matching the classes' packages structure will be created.
202      * @param sourclet The Sourclet to use for formatting the output.
203      * @throws IOException if an i/o error occurs
204      */
205     public static void buildSource(Package p, File dir, File[] sources, Sourclet sourclet) throws IOException {
206         // classes
207 NamedIterator it=p.getClasses();
208         while (it.hasMore()) {
209             Class c=(Class )it.next();
210             if ( (sources == null) || (c.getSource()==null) || isInSources(new File(c.getSource()), sources) ) { // if originating from files, suppress building those sources that have only been loaded for referencing classes, but not been specified as inputs
211 String classname=c.getUnqualifiedName();
212                 File file=new File(dir,classname+".java");
213                 log("writing "+file.getPath());
214                 FileOutputStream f=new FileOutputStream(file);
215                 BufferedOutputStream buf = new BufferedOutputStream(f); // this might cause little optimization, as we are writing many small bits in sequence to the stream
216 sourclet.buildSource(buf,c);
217                 buf.close();
218             }
219         }
220
221         // inner packages
222 it=p.getInnerPackages();
223         while (it.hasMore()) {
224             Package pp=(Package )it.next();
225             String pname=pp.getUnqualifiedName();
226             File indir=new File(dir,pname);
227             boolean created=indir.mkdir();
228             if (created) {
229                 log("directory "+indir.getPath()+" created");
230             }
231             buildSource(pp,indir,sources, sourclet);
232         }
233     }
234
235     /**
236      * Tool function: indent a multi-line string by <i>depth</i> blank characters in front of each line.
237      */
238     public static String indent(String s, int depth) {
239         StringBuffer sb=new StringBuffer ();
240         StringTokenizer st=new StringTokenizer(s,"\n");
241         while (st.hasMoreTokens()) {
242             sb.append(repeat(" ",depth)+st.nextToken()+(st.hasMoreTokens()?"\n":""));
243         }
244         return sb.toString();
245     }
246
247     /**
248      * Tool function: create a new String which contains <i>s</i> repeated <i>c</i> times.
249      */
250     public static String repeat(String s, int c) {
251         if (c>0) {
252             StringBuffer sb=new StringBuffer (s);
253             for (int i=1;i<c;i++) {
254                 sb.append(s);
255             }
256             return sb.toString();
257         }
258         else {
259             return "";
260         }
261     }
262
263     /**
264      * Tool function: replace any occurrence of <i>old</i> in <i>s</i> with <i>neu</i>.
265      */
266     public static String replace(String s, String old, String neu) {
267         int pos=s.indexOf(old);
268         if (pos!=-1) {
269             return s.substring(0,pos)+neu+replace(s.substring(pos+old.length()),old,neu);
270         }
271         else {
272             return s;
273         }
274     }
275
276     /**
277      * Restores manipualted Java source code which avoided single-char unicode characters
278      * back to the original code.
279      * Called only from Code.java.
280      *
281      * @param s manipulated Java source string, as returned from workaroundAvoidUnicodeSingleChar()
282      * @return the original Java source code, as it had been passed as input to workaroundAvoidUnicodeSingleChar()
283      * @see #workaroundAvoidUnicodeSingleChar(String)
284      */
285     public static String workaroundRestoreUnicodeSingleChar(String s) {
286         int pos = s.indexOf("\"" + workaroundUnicodeSingleCharMarker);
287         int l = workaroundUnicodeSingleCharMarker.length();
288         if (pos != -1) {
289             return s.substring(0, pos) + "'\\u" + s.substring(pos+(l+1), pos+(l+5)) + "'" + workaroundRestoreUnicodeSingleChar(s.substring(pos+(l+6)));
290         } else {
291             return s;
292         }
293     }
294
295     /**
296      * Creates XML document builder.
297      */
298     protected static DocumentBuilder getDocumentBuilder() {
299         if (documentBuilder==null) {
300             // init on demand
301 DocumentBuilderFactory dbf=DocumentBuilderFactory.newInstance();
302             dbf.setIgnoringComments(true);
303             dbf.setExpandEntityReferences(validateXML);
304             dbf.setValidating(validateXML); // seems to have no effect, always true (?)
305 try {
306                 documentBuilder=dbf.newDocumentBuilder();
307             }
308             catch (ParserConfigurationException pce) {
309                 System.out.println("ERROR: cannot initialize XML parser - "+pce.getMessage());
310                 // program will exit with NullPointerException after return from this method
311 }
312         }
313         return documentBuilder;
314     }
315
316     /**
317      *
318      * @throws IOException if an i/o error occurs
319      */
320     protected static void parsePass1(Package basePackage, File[] files, ProgressTracker pt) throws IOException, ParseException {
321         Vector todoFiles=new Vector();
322         for (int i=0;i<files.length;i++) {
323             if (files[i].exists()) {
324                 if (files[i].isFile()) {
325                     String filename=files[i].getName();
326                     if (filename.endsWith(".java")) {
327                         if (pt!=null) {
328                             pt.todo(4);
329                         }
330                         todoFiles.addElement(files[i]);
331                     }
332                     else {
333                         //nop, ignore other file types
334 }
335                 }
336                 else if (files[i].isDirectory()) {
337                     String [] list=files[i].list();
338                     File[] ff=new File[list.length];
339                     for (int j=0;j<list.length;j++) {
340                         File ffile=new File(files[i],list[j]);
341                         ff[j]=ffile;
342                     }
343                     parsePass1(basePackage,ff,pt); //recursion
344 }
345             }
346             else {
347                 warning("warning: can't find input file/directory "+files[i].getPath()+", ignoring");
348             }
349         }
350
351         for (Enumeration e=todoFiles.elements();e.hasMoreElements();) {
352             File f=(File)e.nextElement();
353             log("parsing pass 1: "+f.getPath());
354             String code = readFile(f);
355             // workaround 1: avoid \r
356 code = code.replace('\r', ' ');
357             // workaround 2: parser would resolve unicode character declarations ' \ u xxxx ', so change them to a pseudo-string (this is quite an ugly workaround...)
358 // need to call workarondUnicodeRestore() after parsing code blocks
359 // Another workaround for a parser bug: if last line of a source file is a single-line-comment without ending line-break, the parser will crash
360 // so append a 'safety-linefeed' after each input file, it can't do any harm (see JavaCC FAQ 3.15)
361 code = workaroundAvoidUnicodeSingleChar(code) + nl;
362             InputStream in = new StringBufferInputStream(code);
363             analysePass1(basePackage, in, f.getAbsolutePath(), pt);
364         }
365     }
366
367     /**
368      * Parsing pass 1.<br>
369      * This calls the parser generated by JavaCC and converts input source
370      * code into an object-structure.<br>
371      *
372      * @throws IOException if an i/o error occurs
373      * @see #analysePass2
374      */
375     protected static void analysePass1(Package basePackage, InputStream in, String source, ProgressTracker pt) throws IOException, ParseException {
376         Node rootnode;
377         Node[] nodes;
378
379         // syntax parse
380 rootnode=Parser.parse(in,source);
381
382         if (pt!=null) {
383             pt.done(3); // pass1 weighted as 3, pass2 as 1
384 }
385
386         // semantic analysis: create class structure
387
388         // package
389 Package pakkage;
390         Node pak=rootnode.getChild(JJT_PACKAGE);
391         if (pak!=null) {
392             pakkage=new Package ();
393             pakkage.initFromAST(pak);
394         }
395         else {
396             pakkage=basePackage;
397         }
398
399         // imports
400 nodes=rootnode.getChildren(JJT_IMPORT);
401         Vector imports=new Vector();
402         for (int i=0;i<nodes.length;i++) {
403             imports.addElement(Import.createFromAST(basePackage,nodes[i]));
404         }
405
406         // interfaces
407 nodes=rootnode.getChildren(JJT_INTERFACE);
408         for (int i=0;i<nodes.length;i++) {
409             Class c=new Class ();
410             c.setPackage(pakkage);
411             NamedIterator it=c.getImports();
412             for (Enumeration e=imports.elements();e.hasMoreElements();) {
413                 Import im=(Import)e.nextElement();
414                 it.add(im);
415             }
416             c.setInterface(true);
417             c.initFromAST(nodes[i]); // pass 1 only
418 }
419
420         // classes
421 nodes=rootnode.getChildren(JJT_CLASS);
422         for (int i=0;i<nodes.length;i++) {
423             Class c=new Class ();
424             c.setPackage(pakkage);
425             NamedIterator it=c.getImports();
426             for (Enumeration e=imports.elements();e.hasMoreElements();) {
427                 Import im=(Import)e.nextElement();
428                 it.add(im);
429             }
430             c.initFromAST(nodes[i]); // pass 1 only
431 }
432
433         basePackage.add(pakkage);
434     }
435
436     /**
437      * Parsing pass 2.<br>
438      * Now that all classes in packages are already known,
439      * unqualified references can be qualified clearly.
440      * So let classes/ interfaces perform their 'real' initialization now.
441      *
442      * @throws IOException if an i/o error occurs
443      * @see #analysePass1
444      */
445     protected static void analysePass2(Package pack, ProgressTracker pt) throws IOException, ParseException {
446         NamedIterator it;
447         it=pack.getClasses();
448         while (it.hasMore()) {
449             Class c=(Class )it.next();
450             if (!c.pass2) { // (ask here to suppress message)
451 log("parsing pass 2: " + c.getName());
452                 c.initFromASTPass2();
453             }
454             if (pt!=null) {
455                 pt.done(1); // pass1 weighted as 3, pass2 as 1
456 }
457         }
458         // perform this recursively on all inner packages
459 it=pack.getInnerPackages();
460         while (it.hasMore()) {
461             Package p=(Package )it.next();
462             analysePass2(p,pt);
463         }
464     }
465
466     /**
467      * Replaces all occurrences of single-character-constants using unicode
468      * with a pseudo-string. This way, the parser does not resolve the unicode char.
469      * This is quite an ugly workaround, but usually not too costy, as single unicode chars
470      * are rarely used.
471      *
472      * @param s Java source code string, maybe containg single-char unicode constants.
473      * @return manipulated Java source string
474      * @see #workaroundRestoreUnicodeSingleChar(String)
475      */
476     protected static String workaroundAvoidUnicodeSingleChar(String s) {
477         int pos = s.indexOf("'\\u");
478         if (pos != -1) {
479             // make sure this is not inside a string constant
480 int linestart = s.lastIndexOf(nl, pos) + 1; // will result in 0 for 'not found' which is wanted
481 char q = endsQuoted(s.substring(linestart, pos));
482             if (q ==(char)0) {
483                 return s.substring(0, pos) + "\"" + workaroundUnicodeSingleCharMarker + s.substring(pos+3, pos+7) + "\"" + workaroundAvoidUnicodeSingleChar(s.substring(pos+8));
484             } else {
485                 int qe = quoteEnd(s, pos, q);
486                 return s.substring(0, qe+1) + workaroundAvoidUnicodeSingleChar(s.substring(qe+1));
487             }
488         } else {
489             return s;
490         }
491     }
492
493     /**
494      *
495      * @throws IOException if an i/o error occurs
496      */
497     protected static String readFile(File f) throws IOException {
498         FileReader r = new FileReader(f);
499         char[] c = new char[(int)f.length()];
500         r.read(c);
501         r.close();
502         return new String (c);
503     }
504
505     protected static char endsQuoted(String s) {
506         char[] cc = new char[s.length()];
507         s.getChars(0, cc.length, cc, 0);
508         boolean escaped = false;
509         char quoted = (char)0;
510
511         for (int i=0; i<cc.length;i++) {
512             char c = cc[i];
513             if (escaped) {
514                 escaped = false;
515             } else {
516                 switch (c) {
517                     case '\\': escaped = true;
518                                    break;
519                        case '\"': switch (quoted) {
520                                         case '\'': break;
521                                         case '\"': quoted = (char)0; // unquote again
522 break;
523                                         default: quoted = '\"';
524                                                        break;
525                                    }
526                                     break;
527                        case '\'': switch (quoted) {
528                             case '\"': break;
529                             case '\'': quoted = (char)0; // unquote again
530 break;
531                             default: quoted = '\'';
532                                     break;
533                                     }
534                                     break;
535                        }
536                 }
537             }
538         return quoted;
539     }
540
541     protected static int quoteEnd(String s, int pos, char quoteChar) {
542         boolean escaped = false;
543         while (pos < s.length()) {
544             if (escaped) {
545                 escaped = false;
546             } else {
547                 char c = s.charAt(pos);
548                 if (c == quoteChar) {
549                     return pos;
550                 } else if (c == '\\') {
551                     escaped = true;
552                 }
553             }
554             pos++;
555         }
556         return -1;
557     }
558
559     /**
560      * Outputs a log message if the verbose-flag is set..
561      *
562      * @param msg The log message string.
563      */
564     protected static void log(String msg) {
565         if (verbose) {
566             logPerformer.log(msg);
567         }
568     }
569
570     /**
571      * Outputs a warning message, which is the same as outputting a log message, but is performed even is verbose==false.
572      *
573      * @param msg The warning message string.
574      */
575     protected static void warning(String msg) {
576         logPerformer.log(msg);
577     }
578
579     private static boolean isInSources(File f, File[] sources) {
580         try {
581             String fc = f.getCanonicalPath();
582             for (int i=0; i<sources.length; i++) {
583                 String ffc = sources[i].getCanonicalPath();
584                 if (fc.startsWith(ffc)) { // or equal
585 return true;
586                 }
587             }
588             return false;
589         } catch (IOException ioe) {
590             return false;
591         }
592     }
593
594 } // end SourceParser
595

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags