Java > Open Source Codes > org > netbeans > core > modules > ModuleInitTest


1 /*
2  * The contents of this file are subject to the terms of the Common Development
3  * and Distribution License (the License). You may not use this file except in
4  * compliance with the License.
5  *
6  * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7  * or http://www.netbeans.org/cddl.txt.
8  *
9  * When distributing Covered Code, include this CDDL Header Notice in each file
10  * and include the License file at http://www.netbeans.org/cddl.txt.
11  * If applicable, add the following below the CDDL Header, with the fields
12  * enclosed by brackets [] replaced by your own identifying information:
13  * "Portions Copyrighted [year] [name of copyright owner]"
14  *
15  * The Original Software is NetBeans. The Initial Developer of the Original
16  * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
17  * Microsystems, Inc. All Rights Reserved.
18  */
19
20 // To run me:
21 // cd nb_all
22 // ant -f nbbuild/build.xml
23 // java -classpath core/netbeans/lib/ext/boot.jar:core/netbeans/lib/core.jar:openide/netbeans/lib/openide.jar:performance/src:junit/external/junit-3.7.jar:core/test/perf/src:core/external/xerces-2.0.2.jar:core/external/xml-apis-1.0b2.jar org.netbeans.core.modules.ModuleInitTest
24
25 package org.netbeans.core.modules;
26
27 import java.io.*;
28 import java.lang.reflect.Method ;
29 import java.text.DecimalFormat ;
30 import java.util.*;
31 import java.util.zip.*;
32 import java.util.jar.*;
33
34 import org.w3c.dom.*;
35 import org.apache.xml.serialize.*;
36
37 import org.netbeans.core.NbTopManager;
38 import org.netbeans.performance.Benchmark;
39 import org.openide.xml.XMLUtil;
40
41 // initial run (04-sep-02) with empty dummy modules under JDK 1.4.1rc:
42 // 0 - 0.92Mb, 2.75s
43 // 10 - 1.44Mb, 3.36s
44 // 100 - 1.71Mb, 4.5s
45 // 1000 - 7.21Mb, 9-12s
46
47 // another run (04-sep-02) with module dependencies:
48 // 0 - 0.92Mb, 2.77s
49 // 10 - 1.41Mb, 3.37s
50 // 100 - 1.69Mb, 4.3s
51 // 1000 - 7.72Mb, 10-12s
52 // So dependencies only seem to have a small startup time impact
53 // for a lot of modules (hundreds, a second or so).
54
55 // 7ms/module to read manifest & .xml file, plus .25ms/module to get file list from Modules/ (mostly file access?)
56 // .35ms/module to check dependency & orderings
57 // 2.4ms/module to open JAR & create classloader
58 // .2ms/m to look for nonexistent sections [already improving]
59 // .14ms/m to look for nonexistent module installs [how to improve?]
60
61 // after some tweaks (05-sep-02), measured inside NB w/ term:
62 // 0 - 0.87Mb, 3.0s
63 // 10 - 1.32Mb, 3.7s
64 // 100 - 1.73Mb, 4.6s
65 // 1000 - 6.29Mb, 11.3s
66 // measured outside from a shell:
67 // 0 - 1.15Mb, 2.9s
68 // 10 - 0.82Mb, 3.1s
69 // 100 - 1.66Mb, around 4s
70 // 1000 - 5.74Mb, 10.5s
71 // after adding 1000 files to each module:
72 // 1000 - 6.82Mb, 13.2s
73 // after adding 0 - 250 files to each module:
74 // 1000 - 6.64Mb, around 12s
75 // after adding 23 files to each module's layer:
76 // 1000 - 19.00Mb, around 17s
77 // with some attributes too, 2 on 1/5 of folders and 1 on 1/3 of files:
78 // 1000 - 20.3Mb, around 17.5s
79
80 // so about 5.5ms/m for layer parsing, <1ms/m for opening a bigger JAR
81
82 // After ModuleList opt to not use Crimson to parse Modules/*.xml:
83 // 1000 - 21.1Mb, 16.4s
84 // detail timing: now only 3.8ms/m to load manifest + parse XML
85
86 /**
87  * Benchmark measuring initialization of the module system.
88  * Covers parsing of module config XML files; opening JARs;
89  * reading their manifests; computing dependencies; loading XML layers;
90  * turning everything on.
91  * @author Jesse Glick
92  * @see "#26786"
93  */
94 public class ModuleInitTest extends Benchmark {
95     
96     public static void main(String [] args) {
97         simpleRun(ModuleInitTest.class);
98     }
99     
100     private static Map[] parseParams(String [] ps) {
101         Map[] m = new Map[ps.length];
102         for (int i = 0; i < ps.length; i++) {
103             m[i] = new HashMap(); // Map<String,String|Integer>
104 StringTokenizer tok = new StringTokenizer(ps[i], ",");
105             while (tok.hasMoreTokens()) {
106                 String kv = tok.nextToken();
107                 int x = kv.indexOf('=');
108                 String k = kv.substring(0, x);
109                 String v = kv.substring(x + 1);
110                 try {
111                     m[i].put(k, new Integer (v));
112                 } catch (NumberFormatException nfe) {
113                     m[i].put(k, v);
114                 }
115             }
116         }
117         return m;
118     }
119     
120     public ModuleInitTest(String name) {
121         super(name, parseParams(new String [] {
122             // Simple scalability test:
123 "modules=0,jarSize=200,layerSize=23",
124             "modules=10,jarSize=200,layerSize=23",
125             "modules=100,jarSize=200,layerSize=23",
126             "modules=1000,jarSize=200,layerSize=23",
127             /* Test manifest caching:
128             "modules=100,jarSize=200,layerSize=0,-Dnetbeans.cache.manifests=false",
129             "modules=100,jarSize=200,layerSize=0,-Dnetbeans.cache.manifests=true",
130             "modules=1000,jarSize=200,layerSize=0,-Dnetbeans.cache.manifests=false",
131             "modules=1000,jarSize=200,layerSize=0,-Dnetbeans.cache.manifests=true",
132             */
133         }));
134     }
135     
136     private static File getTmpDir() {
137         File ramdisk = new File("/dev/shm");
138         if (ramdisk.isDirectory() && ramdisk.canWrite()) {
139             return ramdisk;
140         } else {
141             return new File(System.getProperty("java.io.tmpdir"));
142         }
143     }
144     private File topdir = new File(getTmpDir(), "ModuleInitTest");
145     private File homedir = new File(topdir, "home");
146     private File skeldir = new File(topdir, "skeluser");
147     private File userdir = new File(topdir, "user");
148     private Map lastParams = null;
149     
150     protected void setUp() throws Exception {
151         Map params = (Map)getArgument();
152         if (!params.equals(lastParams)) {
153             //System.out.println("Setup: " + params);
154 if (homedir.exists()) {
155                 deleteRec(homedir);
156             }
157             File mods = new File(homedir, "modules");
158             File amods = new File(mods, "autoload");
159             amods.mkdirs();
160             File emods = new File(mods, "eager");
161             emods.mkdirs();
162             createModules(params, mods, amods, emods);
163             new File(homedir, "system").mkdirs();
164             // Priming run to create system/Modules directory:
165 if (skeldir.exists()) {
166                 deleteRec(skeldir);
167             }
168             runNB(homedir, skeldir, false, params);
169             lastParams = params;
170         }
171         // On every run, copy the primed skeleton user dir to the real location,
172 // then start NB with the copied user dir.
173 if (userdir.exists()) {
174             deleteRec(userdir);
175         }
176         copyRec(skeldir, userdir);
177     }
178     protected void tearDown() throws Exception {
179     }
180     
181     private static void deleteRec(File x) throws IOException {
182         File[] kids = x.listFiles();
183         if (kids != null) {
184             for (int i = 0; i < kids.length; i++) {
185                 deleteRec(kids[i]);
186             }
187         }
188         if (!x.delete()) throw new IOException("Could not delete: " + x);
189     }
190     private static void copyStream(InputStream is, OutputStream os) throws IOException {
191         try {
192             byte[] b = new byte[4096];
193             int i;
194             while ((i = is.read(b)) != -1) {
195                 os.write(b, 0, i);
196             }
197         } finally {
198             is.close();
199         }
200     }
201     private static void copyRec(File x, File y) throws IOException {
202         if (x.isDirectory()) {
203             if (!y.mkdirs()) throw new IOException("Could not mkdir: " + y);
204             String [] kids = x.list();
205             if (kids == null) throw new IOException("Could not list: " + x);
206             for (int i = 0; i < kids.length; i++) {
207                 copyRec(new File(x, kids[i]), new File(y, kids[i]));
208             }
209         } else {
210             y.getParentFile().mkdirs();
211             OutputStream os = new FileOutputStream(y);
212             try {
213                 copyStream(new FileInputStream(x), os);
214             } finally {
215                 os.close();
216             }
217         }
218     }
219     
220     // Sorry, I can't easily draw this dependency graph.
221 // It is complicated - that is the whole point.
222 private static final int cyclesize = 8;
223     // Base names of cyclic pattern of modules:
224 private static final String [] names = {"aut1", "prv1", "reg1", "aut2", "reg2", "reg3", "dis1", "eag1"};
225     // Types: 0 = regular, 1 = autoload, 2 = eager
226 private static final int[] types = {1, 1, 0, 1, 0, 0, 0, 2};
227     // Intra-set dependencies (list of indices of other modules):
228 private static final int[][] intradeps = {{}, {}, {0}, {}, {0, 3}, {3}, {3}, {4, 5}};
229     // Inter-set dependencies (list of indices of analogous modules in the previous cycle):
230 private static final int[][] interdeps = {{0}, {}, {2}, {}, {}, {}, {6}, {5}};
231     // Whether the module should be permitted to be enabled or not:
232 private static final boolean[] enabled = {true, true, true, true, true, true, false, true};
233     // Provided tokens from the module; freeform, but if ends in '#' that will get subst'd w/ cycle
234 private static final String [][] provides = {{}, {"tok#"}, {}, {}, {}, {}, {}, {}};
235     // Required tokens; same syntax as above.
236 private static final String [][] requires = {{}, {}, {"tok#"}, {}, {}, {}, {}, {}};
237     // E.g.: module #16 (0-indexed) is in the third cycle and is a reg1, thus named reg1_002.
238 // It depends on module #14, aut1_002 (from intradeps) and module #9, reg1_001 (from interdeps).
239 // It also depends on tok_002, provided only by module #15, prv1_002.
240 // All of these will be enabled, along with some other modules (indirectly).
241 static {
242         if (names.length != cyclesize ||
243                 types.length != cyclesize ||
244                 intradeps.length != cyclesize ||
245                 interdeps.length != cyclesize ||
246                 enabled.length != cyclesize ||
247                 provides.length != cyclesize ||
248                 requires.length != cyclesize) {
249             throw new Error ();
250         }
251     }
252     
253     private void createModules(Map params, File mods, File amods, File emods) throws IOException {
254         int size = ((Integer )params.get("modules")).intValue();
255         int jarSize = ((Integer )params.get("jarSize")).intValue();
256         int layerSize = ((Integer )params.get("layerSize")).intValue();
257         File[] moddirs = {mods, amods, emods}; // indexed by types[n]
258 for (int i = 0; i < size; i++) {
259             int which = i % cyclesize;
260             int cycle = i / cyclesize;
261             String cycleS = Integer.toString(cycle);
262             // Enough for 1000*cyclesize modules to sort nicely:
263 while (cycleS.length() < 3) cycleS = "0" + cycleS;
264             Manifest mani = new Manifest();
265             Attributes attr = mani.getMainAttributes();
266             attr.putValue("Manifest-Version", "1.0");
267             String nameBase = names[which] + "_" + cycleS;
268             String name = "com.testdomain." + nameBase;
269             String nameSlashes = name.replace('.', '/');
270             attr.putValue("OpenIDE-Module", name + "/1");
271             // Avoid requiring javahelp:
272 attr.putValue("OpenIDE-Module-IDE-Dependencies", "IDE/1 > 2.2");
273             attr.putValue("OpenIDE-Module-Specification-Version", "1.0");
274             StringBuffer deps = null;
275             for (int j = 0; j < intradeps[which].length; j++) {
276                 if (deps == null) {
277                     deps = new StringBuffer (1000);
278                 } else {
279                     deps.append(", ");
280                 }
281                 deps.append("com.testdomain." + names[intradeps[which][j]]);
282                 deps.append('_');
283                 deps.append(cycleS);
284                 deps.append("/1 > 1.0");
285             }
286             if (cycle > 0) {
287                 String oldCycleS = Integer.toString(cycle - 1);
288                 while (oldCycleS.length() < 3) oldCycleS = "0" + oldCycleS;
289                 for (int j = 0; j < interdeps[which].length; j++) {
290                     if (deps == null) {
291                         deps = new StringBuffer (1000);
292                     } else {
293                         deps.append(", ");
294                     }
295                     deps.append("com.testdomain." + names[interdeps[which][j]]);
296                     deps.append('_');
297                     deps.append(oldCycleS);
298                     deps.append("/1 > 1.0");
299                 }
300             }
301             if (!enabled[which]) {
302                 if (deps == null) {
303                     deps = new StringBuffer (1000);
304                 } else {
305                     deps.append(", ");
306                 }
307                 // An impossible dependency.
308 deps.append("honest.man.in.washington");
309             }
310             if (deps != null) {
311                 attr.putValue("OpenIDE-Module-Module-Dependencies", deps.toString());
312             }
313             if (provides[which].length > 0) {
314                 StringBuffer buf = new StringBuffer (100);
315                 for (int j = 0; j < provides[which].length; j++) {
316                     if (j > 0) {
317                         buf.append(", ");
318                     }
319                     String tok = provides[which][j];
320                     if (tok.endsWith("#")) {
321                         tok = tok.substring(0, tok.length() - 1) + "_" + cycleS;
322                     }
323                     buf.append("com.testdomain." + tok);
324                 }
325                 attr.putValue("OpenIDE-Module-Provides", buf.toString());
326             }
327             if (requires[which].length > 0) {
328                 StringBuffer buf = new StringBuffer (100);
329                 for (int j = 0; j < requires[which].length; j++) {
330                     if (j > 0) {
331                         buf.append(", ");
332                     }
333                     String tok = requires[which][j];
334                     if (tok.endsWith("#")) {
335                         tok = tok.substring(0, tok.length() - 1) + "_" + cycleS;
336                     }
337                     buf.append("com.testdomain." + tok);
338                 }
339                 attr.putValue("OpenIDE-Module-Requires", buf.toString());
340             }
341             // Files in JAR other than manifest:
342 Map contents = new TreeMap(); // Map<String,byte[]>
343 String locb = nameSlashes + "/Bundle.properties";
344             contents.put(locb, ("OpenIDE-Module-Name=Module #" + i + "\n").getBytes());
345             attr.putValue("OpenIDE-Module-Localizing-Bundle", locb);
346             contents.put(nameSlashes + "/foo", "stuff here\n".getBytes());
347             contents.put(nameSlashes + "/subdir/foo", "more stuff here\n".getBytes());
348             for (int j = 0; j < jarSize; j++) {
349                 contents.put(nameSlashes + "/sub/subdir/file" + j, new byte[j]);
350             }
351             // XML layer:
352 Map layer = new TreeMap(); // Map<String,byte[]|null>
353 // Construct a binomial tree, module #i contributing the next layerSize files (leaves):
354 int start = i * layerSize;
355             int end = (i + 1) * layerSize;
356             for (int j = start; j < end; j++) {
357                 String filename = "file" + j + ".txt";
358                 int bit = 0;
359                 int x = j;
360                 while (x > 0) {
361                     if (x % 2 == 1) {
362                         filename = "dir" + bit + "/" + filename;
363                     }
364                     bit++;
365                     x /= 2;
366                 }
367                 layer.put(filename, (j % 2 == 0) ? ("Contents #" + j + "\n").getBytes() : null);
368             }
369             ByteArrayOutputStream baos = new ByteArrayOutputStream(layer.size() * 100 + 1);
370             Document doc = XMLUtil.createDocument("filesystem", null, "-//NetBeans//DTD Filesystem 1.1//EN", "http://www.netbeans.org/dtds/filesystem-1_1.dtd");
371             doc.getDocumentElement().appendChild(doc.createComment(" Layer filenames for module #" + i + ": " + layer.keySet() + " "));
372             Iterator it = layer.entrySet().iterator();
373             while (it.hasNext()) {
374                 Map.Entry e = (Map.Entry)it.next();
375                 String filename = (String )e.getKey();
376                 byte[] filebytes = (byte[])e.getValue();
377                 Element el = doc.getDocumentElement();
378                 StringTokenizer tok = new StringTokenizer(filename, "/");
379                 while (tok.hasMoreTokens()) {
380                     String piece = tok.nextToken();
381                     if (tok.hasMoreTokens()) {
382                         // A dir. Look for it...
383 Element child = null;
384                         NodeList kids = el.getChildNodes();
385                         for (int j = 0; j < kids.getLength(); j++) {
386                             Node n = kids.item(j);
387                             if (!(n instanceof Element)) continue;
388                             Element kid = (Element)n;
389                             if (!kid.getNodeName().equals("folder")) continue;
390                             if (kid.getAttribute("name").equals(piece)) {
391                                 child = kid;
392                                 break;
393                             }
394                         }
395                         if (child == null) {
396                             child = doc.createElement("folder");
397                             child.setAttribute("name", piece);
398                             // Attributes, on every fifth folder:
399 if (Math.abs(filename.hashCode()) % 5 == 0) {
400                                 Element a = doc.createElement("attr");
401                                 a.setAttribute("name", "SystemFileSystem.localizingBundle");
402                                 a.setAttribute("stringvalue", name + ".Bundle");
403                                 child.appendChild(a);
404                                 a = doc.createElement("attr");
405                                 a.setAttribute("name", "SystemFileSystem.icon");
406                                 a.setAttribute("urlvalue", "nbresloc:/" + nameSlashes + "/resources/" + piece + ".gif");
407                                 child.appendChild(a);
408                             }
409                             el.appendChild(child);
410                         }
411                         el = child;
412                     } else {
413                         // This is a file. It is not already in the doc - because
414 // of the map.
415 Element child = doc.createElement("file");
416                         child.setAttribute("name", piece);
417                         if (filebytes != null) {
418                             String contentsName = "resources/layerfile" + Integer.toHexString(filename.hashCode());
419                             contents.put(nameSlashes + "/" + contentsName, filebytes);
420                             child.setAttribute("url", contentsName);
421                         }
422                         // Attributes, on every third file:
423 if (Math.abs(filename.hashCode()) % 3 == 0) {
424                             Element a = doc.createElement("attr");
425                             a.setAttribute("name", "instanceOf");
426                             a.setAttribute("stringvalue", name + ".Whatever");
427                             child.appendChild(a);
428                         }
429                         el.appendChild(child);
430                     }
431                 }
432             }
433             XMLSerializer ser = new XMLSerializer(baos, new OutputFormat(doc, "UTF-8", true));
434             ser.serialize(doc);
435             String layerName = nameSlashes + "/layer.xml";
436             contents.put(layerName, baos.toByteArray());
437             attr.putValue("OpenIDE-Module-Layer", layerName);
438             OutputStream os = new FileOutputStream(new File(moddirs[types[which]], nameBase + ".jar"));
439             try {
440                 JarOutputStream jos = new JarOutputStream(os, mani);
441                 Set addedDirs = new HashSet(1000); // Set<String>
442 it = contents.entrySet().iterator();
443                 while (it.hasNext()) {
444                     Map.Entry e = (Map.Entry)it.next();
445                     String filename = (String )e.getKey();
446                     byte[] filebytes = (byte[])e.getValue();
447                     String dircheck = filename;
448                     while (true) {
449                         int idx = dircheck.lastIndexOf('/');
450                         if (idx == -1) break;
451                         dircheck = dircheck.substring(0, idx);
452                         if (!addedDirs.add(dircheck)) break;
453                         JarEntry je = new JarEntry(dircheck + "/");
454                         je.setMethod(ZipEntry.STORED);
455                         je.setSize(0);
456                         je.setCompressedSize(0);
457                         je.setCrc(0);
458                         jos.putNextEntry(je);
459                     }
460                     JarEntry je = new JarEntry(filename);
461                     je.setMethod(ZipEntry.DEFLATED);
462                     //je.setSize(filebytes.length);
463 jos.putNextEntry(je);
464                     jos.write(filebytes);
465                 }
466                 jos.close();
467             } finally {
468                 os.close();
469             }
470         }
471     }
472     
473     public void testInitModuleSystem() throws Exception {
474         int count = getIterationCount();
475         for (int i = 0; i < count; i++) {
476             runNB(homedir, userdir, true, (Map)getArgument());
477         }
478     }
479     
480     private String cp = refinecp(System.getProperty("java.class.path"));
481     
482     private void runNB(File homedir, File userdir, boolean log, Map params) throws IOException {
483         List cmd = new ArrayList(Arrays.asList(new String [] {
484             "java",
485             "-Xms24m",
486             "-Xmx96m",
487             "-Dorg.openide.TopManager=org.netbeans.core.NonGuiMain",
488             "-Dnetbeans.security.nocheck=true",
489             "-Dnetbeans.home=" + homedir.getAbsolutePath(),
490             "-Dnetbeans.user=" + userdir.getAbsolutePath(),
491             "-Dnetbeans.modules.quiet=true",
492             "-Dlog=" + log,
493             "-classpath",
494             cp,
495         }));
496         Iterator it = params.entrySet().iterator();
497         while (it.hasNext()) {
498             Map.Entry e = (Map.Entry)it.next();
499             String key = (String )e.getKey();
500             if (key.startsWith("-D")) {
501                 cmd.add(key + "=" + e.getValue());
502             }
503         }
504         //if (log) cmd.add("-Dorg.netbeans.log.startup=print");
505 cmd.add("org.netbeans.core.modules.ModuleInitTest$Main");
506         //System.out.println("Running: " + cmd);
507 Process p = Runtime.getRuntime().exec((String [])cmd.toArray(new String [cmd.size()]));
508         new Copier(p.getInputStream(), System.out).start();
509         new Copier(p.getErrorStream(), System.err).start();
510         try {
511             int stat = p.waitFor();
512             if (stat != 0) {
513                 throw new IOException("Command failed (status " + stat + "): " + cmd);
514             }
515         } catch (InterruptedException ie) {
516             throw new IOException(ie.toString());
517         }
518     }
519     
520     /** Remove openide/test/perf/src/ if present since it overrides ErrorManager badly.
521      */
522     private static String refinecp(String cp) {
523         StringBuffer b = new StringBuffer (cp.length());
524         StringTokenizer t = new StringTokenizer(cp, File.pathSeparator);
525         while (t.hasMoreTokens()) {
526             File f = new File(t.nextToken());
527             if (f.isDirectory()) {
528                 if (new File(new File(new File(f, "org"), "openide"), "ErrorManagerTest.java").exists() ||
529                         new File(new File(new File(f, "org"), "openide"), "ErrorManagerTest.class").exists()) {
530                     //System.err.println("Removing " + f + " from classpath");
531 continue;
532                 }
533             }
534             if (b.length() != 0) {
535                 b.append(File.pathSeparatorChar);
536             }
537             b.append(f);
538         }
539         return b.toString();
540     }
541     
542     private static final class Copier extends Thread {
543         private final InputStream is;
544         private final PrintStream ps;
545         public Copier(InputStream is, PrintStream ps) {
546             this.is = is;
547             this.ps = ps;
548         }
549         public void run() {
550             try {
551                 copyStream(is, ps);
552             } catch (IOException ioe) {
553                 ioe.printStackTrace();
554             }
555         }
556     }
557     
558     public static final class Main {
559         public static void main(String [] x) {
560             NbTopManager.get();
561             if (Boolean.getBoolean("log")) {
562                 Runtime r = Runtime.getRuntime();
563                 r.gc();
564                 double megs = (r.totalMemory() - r.freeMemory()) / 1024.0 / 1024.0;
565                 System.err.println("Used memory: " + new DecimalFormat ("0.00 Mb").format(megs));
566             }
567         }
568     }
569     
570 }
571

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags