Java > Open Source Codes > com > google > gwt > junit > rebind > BenchmarkGenerator


1 /*
2  * Copyright 2007 Google Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License"); you may not
5  * use this file except in compliance with the License. You may obtain a copy of
6  * the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13  * License for the specific language governing permissions and limitations under
14  * the License.
15  */
16 package com.google.gwt.junit.rebind;
17
18 import com.google.gwt.core.ext.TreeLogger;
19 import com.google.gwt.core.ext.UnableToCompleteException;
20 import com.google.gwt.core.ext.typeinfo.JClassType;
21 import com.google.gwt.core.ext.typeinfo.JMethod;
22 import com.google.gwt.core.ext.typeinfo.JParameter;
23 import com.google.gwt.junit.JUnitShell;
24 import com.google.gwt.dev.generator.ast.ForLoop;
25 import com.google.gwt.dev.generator.ast.MethodCall;
26 import com.google.gwt.dev.generator.ast.Statement;
27 import com.google.gwt.dev.generator.ast.Statements;
28 import com.google.gwt.dev.generator.ast.StatementsList;
29 import com.google.gwt.user.rebind.SourceWriter;
30
31 import java.util.Map ;
32 import java.util.Iterator ;
33 import java.util.List ;
34 import java.util.Set ;
35 import java.util.HashMap ;
36 import java.util.ArrayList ;
37 import java.util.Collections ;
38
39 /**
40  * Implements a generator for Benchmark classes. Benchmarks require additional
41  * code generation above and beyond standard JUnit tests.
42  */
43 public class BenchmarkGenerator extends JUnitTestCaseStubGenerator {
44
45   private static class MutableBoolean {
46     boolean value;
47   }
48
49   private static final String BEGIN_PREFIX = "begin";
50
51   private static final String BENCHMARK_PARAM_META = "gwt.benchmark.param";
52
53   private static final String EMPTY_FUNC = "__emptyFunc";
54
55   private static final String END_PREFIX = "end";
56
57   private static final String ESCAPE_LOOP = "__escapeLoop";
58
59   /**
60    * Returns all the zero-argument JUnit test methods that do not have
61    * overloads.
62    *
63    * @return Map<String,JMethod>
64    */
65   public static Map getNotOverloadedTestMethods(JClassType requestedClass) {
66     Map methods = getAllMethods(requestedClass, new MethodFilter() {
67       public boolean accept(JMethod method) {
68         return isJUnitTestMethod(method, true);
69       }
70     });
71
72     for (Iterator it = methods.entrySet().iterator(); it.hasNext();) {
73       Map.Entry entry = (Map.Entry ) it.next();
74       List methodOverloads = (List) entry.getValue();
75       if (methodOverloads.size() > 1) {
76         it.remove();
77         continue;
78       }
79       entry.setValue(methodOverloads.get(0));
80     }
81
82     return methods;
83   }
84
85   /**
86    * Returns all the JUnit test methods that are overloaded test methods with
87    * parameters. Does not include the zero-argument test methods.
88    *
89    * @return Map<String,JMethod>
90    */
91   public static Map getParameterizedTestMethods(JClassType requestedClass,
92       TreeLogger logger) {
93
94     Map testMethods = getAllMethods(requestedClass, new MethodFilter() {
95       public boolean accept(JMethod method) {
96         return isJUnitTestMethod(method, true);
97       }
98     });
99
100     // Remove all non-overloaded test methods
101 for (Iterator it = testMethods.entrySet().iterator(); it.hasNext();) {
102
103       Map.Entry entry = (Map.Entry ) it.next();
104       String name = (String ) entry.getKey();
105       List methods = (List) entry.getValue();
106
107       if (methods.size() > 2) {
108         String msg = requestedClass + "." + name
109             + " has more than one overloaded version.\n" +
110             "It will not be included in the test case execution.";
111         logger.log(TreeLogger.WARN, msg, null);
112         it.remove();
113         continue;
114       }
115
116       if (methods.size() == 1) {
117         JMethod method = (JMethod) methods.get(0);
118         if (method.getParameters().length != 0) {
119           /* User probably goofed - otherwise why create a test method with
120            * arguments but not the corresponding no-argument version? Would be
121            * better if our benchmarking system didn't require the no-argument
122            * test to make the benchmarks run correctly (JUnit artifact).
123            */
124           String msg = requestedClass + "." + name
125               + " does not have a zero-argument overload.\n" +
126               "It will not be included in the test case execution.";
127           logger.log(TreeLogger.WARN, msg, null);
128         }
129         // Only a zero-argument version, we don't need to process it.
130 it.remove();
131         continue;
132       }
133
134       JMethod method1 = (JMethod) methods.get(0);
135       JMethod method2 = (JMethod) methods.get(1);
136       JMethod noArgMethod = null;
137       JMethod overloadedMethod = null;
138
139       if (method1.getParameters().length == 0) {
140         noArgMethod = method1;
141       } else {
142         overloadedMethod = method1;
143       }
144
145       if (method2.getParameters().length == 0) {
146         noArgMethod = method2;
147       } else {
148         overloadedMethod = method2;
149       }
150
151       if (noArgMethod == null) {
152         String msg = requestedClass + "." + name
153             + " does not have a zero-argument overload.\n" +
154             "It will not be included in the test case execution.";
155         logger.log(TreeLogger.WARN, msg, null);
156         it.remove();
157         continue;
158       }
159
160       entry.setValue(overloadedMethod);
161     }
162
163     return testMethods;
164   }
165
166   private static JMethod getBeginMethod(JClassType type, String name) {
167     StringBuffer methodName = new StringBuffer (name);
168     methodName.replace(0, "test".length(), BEGIN_PREFIX);
169     return getMethod(type, methodName.toString());
170   }
171
172   private static JMethod getEndMethod(JClassType type, String name) {
173     StringBuffer methodName = new StringBuffer (name);
174     methodName.replace(0, "test".length(), END_PREFIX);
175     return getMethod(type, methodName.toString());
176   }
177
178   private static JMethod getMethod(JClassType type, MethodFilter filter) {
179     Map map = getAllMethods(type, filter);
180     Set entrySet = map.entrySet();
181     if (entrySet.size() == 0) {
182       return null;
183     }
184     List methods = (List) ((Map.Entry ) entrySet.iterator().next()).getValue();
185     return (JMethod) methods.get(0);
186   }
187
188   private static JMethod getMethod(JClassType type, final String name) {
189     return getMethod(type, new MethodFilter() {
190       public boolean accept(JMethod method) {
191         return method.getName().equals(name);
192       }
193     });
194   }
195
196   public void writeSource() throws UnableToCompleteException {
197     super.writeSource();
198
199     generateEmptyFunc(getSourceWriter());
200     implementZeroArgTestMethods();
201     implementParameterizedTestMethods();
202     generateAsyncCode();
203     JUnitShell.getReport().addBenchmark(getRequestedClass(), getTypeOracle());
204   }
205
206   /**
207    * Generates benchmarking code which wraps <code>stmts</code> The timing
208    * result is a double in units of milliseconds. It's value is placed in the
209    * variable named, <code>timeMillisName</code>.
210    *
211    * @return The set of Statements containing the benchmark code along with the
212    * wrapped <code>stmts</code>
213    */
214   private Statements benchmark(Statements stmts, String timeMillisName,
215       boolean generateEscape, Statements recordCode, Statements breakCode) {
216     Statements benchmarkCode = new StatementsList();
217     List benchStatements = benchmarkCode.getStatements();
218
219     ForLoop loop = new ForLoop("int numLoops = 1", "true", "");
220     benchStatements.add(loop);
221     List loopStatements = loop.getStatements();
222
223     loopStatements
224         .add(new Statement("long start = System.currentTimeMillis()"));
225     ForLoop runLoop = new ForLoop("int i = 0", "i < numLoops", "++i", stmts);
226     loopStatements.add(runLoop);
227
228     // Put the rest of the code in 1 big statement to simplify things
229 String benchCode =
230         "long duration = System.currentTimeMillis() - start;\n\n" +
231
232         "if ( duration < 150 ) {\n" +
233         " numLoops += numLoops;\n" +
234         " continue;\n" +
235         "}\n\n" +
236
237         "double durationMillis = duration * 1.0;\n" +
238         "double numLoopsAsDouble = numLoops * 1.0;\n" +
239         timeMillisName + " = durationMillis / numLoopsAsDouble";
240
241     loopStatements.add(new Statement(benchCode));
242
243     if (recordCode != null) {
244       loopStatements.add(recordCode);
245     }
246
247     if (generateEscape) {
248       loopStatements.add(new Statement(
249           "if ( numLoops == 1 && duration > 1000 ) {\n" +
250             breakCode.toString() + "\n" +
251           "}\n\n"
252       ));
253     }
254
255     loopStatements.add(new Statement("break"));
256
257     return benchmarkCode;
258   }
259
260   /**
261    * Generates code that executes <code>statements</code> for all possible
262    * values of <code>params</code>. Exports a label named ESCAPE_LOOP that
263    * points to the the "inner loop" that should be escaped to for a limited
264    * variable.
265    *
266    * @return the generated code
267    */
268   private Statements executeForAllValues(JParameter[] methodParams, Map params,
269       Statements statements) {
270     Statements root = new StatementsList();
271     Statements currentContext = root;
272
273     // Profile the setup and teardown costs for this test method
274 // but only if 1 of them exists.
275 for (int i = 0; i < methodParams.length; ++i) {
276       JParameter methodParam = methodParams[i];
277       String paramName = methodParam.getName();
278       String paramValue = (String ) params.get(paramName);
279
280       String iteratorName = "it_" + paramName;
281       String initializer = "java.util.Iterator " + iteratorName + " = "
282           + paramValue + ".iterator()";
283       ForLoop loop = new ForLoop(initializer, iteratorName + ".hasNext()", "");
284       if (i == methodParams.length - 1) {
285         loop.setLabel(ESCAPE_LOOP);
286       }
287       currentContext.getStatements().add(loop);
288       String typeName = methodParam.getType().getQualifiedSourceName();
289       loop.getStatements().add(new Statement(typeName + " " + paramName + " = ("
290           + typeName + ") " + iteratorName + ".next()"));
291       currentContext = loop;
292     }
293
294     currentContext.getStatements().add(statements);
295
296     return root;
297   }
298
299   private Statements genBenchTarget(JMethod beginMethod, JMethod endMethod,
300       List paramNames, Statements test) {
301     Statements statements = new StatementsList();
302     List statementsList = statements.getStatements();
303
304     if (beginMethod != null) {
305       statementsList.add(
306           new Statement(new MethodCall(beginMethod.getName(), paramNames)));
307     }
308
309     statementsList.add(test);
310
311     if (endMethod != null) {
312       statementsList
313           .add(new Statement(new MethodCall(endMethod.getName(), null)));
314     }
315
316     return statements;
317   }
318
319   /**
320    * Currently, the benchmarking subsystem does not support async Benchmarks,
321    * so we need to generate some additional code that prevents the user
322    * from entering async mode in their Benchmark, even though we're using
323    * it internally.
324    *
325    * Generates the code for the "supportsAsync" functionality in the
326    * translatable version of GWTTestCase. This includes:
327    *
328    * - the supportsAsync flag
329    * - the supportsAsync method
330    * - the privateDelayTestFinish method
331    * - the privateFinishTest method
332    *
333    */
334   private void generateAsyncCode() {
335     SourceWriter writer = getSourceWriter();
336
337     writer.println( "private boolean supportsAsync;" );
338     writer.println();
339     writer.println( "public boolean supportsAsync() {");
340     writer.println( " return supportsAsync;");
341     writer.println( "}");
342     writer.println();
343     writer.println( "private void privateDelayTestFinish(int timeout) {" );
344     writer.println( " supportsAsync = true;");
345     writer.println( " try {");
346     writer.println( " delayTestFinish(timeout);");
347     writer.println( " } finally {");
348     writer.println( " supportsAsync = false;");
349     writer.println( " }");
350     writer.println( "}");
351     writer.println();
352     writer.println( "private void privateFinishTest() {" );
353     writer.println( " supportsAsync = true;");
354     writer.println( " try {");
355     writer.println( " finishTest();");
356     writer.println( " } finally {");
357     writer.println( " supportsAsync = false;");
358     writer.println( " }");
359     writer.println( "}");
360     writer.println();
361   }
362
363   /**
364    * Generates an empty JSNI function to help us benchmark function call
365    * overhead.
366    *
367    * We prevent our empty function call from being inlined by the compiler by
368    * making it a JSNI call. This works as of 1.3 RC 2, but smarter versions of
369    * the compiler may be able to inline JSNI.
370    *
371    * Things actually get pretty squirrely in general when benchmarking function
372    * call overhead, because, depending upon the benchmark, the compiler may
373    * inline the benchmark into our benchmark loop, negating the cost we thought
374    * we were measuring.
375    *
376    * The best way to deal with this is for users to write micro-benchmarks such
377    * that the micro-benchmark does significantly more work than a function call.
378    * For example, if micro-benchmarking a function call, perform the function
379    * call 100K times within the microbenchmark itself.
380    */
381   private void generateEmptyFunc(SourceWriter writer) {
382     writer.println("private native void " + EMPTY_FUNC + "() /*-{");
383     writer.println("}-*/;");
384     writer.println();
385   }
386
387   private Map /*<String,String>*/ getParamMetaData(JMethod method,
388       MutableBoolean isBounded) throws UnableToCompleteException {
389     Map /*<String,String>*/ params = new HashMap /*<String,String>*/();
390
391     String [][] allValues = method.getMetaData(BENCHMARK_PARAM_META);
392
393     if (allValues == null) {
394       return params;
395     }
396
397     for (int i = 0; i < allValues.length; ++i) {
398       String [] values = allValues[i];
399       StringBuffer result = new StringBuffer ();
400       for (int j = 0; j < values.length; ++j) {
401         result.append(values[j]);
402         result.append(" ");
403       }
404       String expr = result.toString();
405       String [] lhsAndRhs = expr.split("=");
406       String paramName = lhsAndRhs[0].trim();
407       String [] nameExprs = paramName.split(" ");
408       if (nameExprs.length > 1 && nameExprs[1].equals("-limit")) {
409         paramName = nameExprs[0];
410         // Make sure this is the last parameter
411 JParameter[] parameters = method.getParameters();
412         if (! parameters[parameters.length - 1].getName().equals(paramName)) {
413           JClassType cls = method.getEnclosingType();
414           String msg = "Error at " + cls + "." + method.getName() + "\n" +
415               "Only the last parameter of a method can be marked with the -limit flag.";
416           logger.log(TreeLogger.ERROR, msg, null);
417           throw new UnableToCompleteException();
418         }
419
420         isBounded.value = true;
421       }
422       String paramValue = lhsAndRhs[1].trim();
423       params.put(paramName, paramValue);
424     }
425
426     return params;
427   }
428
429   private void implementParameterizedTestMethods() throws
430       UnableToCompleteException {
431
432     Map /*<String,JMethod>*/ parameterizedMethods = getParameterizedTestMethods(
433         getRequestedClass(), logger);
434     SourceWriter sw = getSourceWriter();
435     JClassType type = getRequestedClass();
436
437     // For each test method, benchmark its:
438 // a) overhead (setup + teardown + loop + function calls) and
439 // b) execution time
440 // for all possible parameter values
441 for (Iterator it = parameterizedMethods.entrySet().iterator();
442         it.hasNext();) {
443       Map.Entry entry = (Map.Entry ) it.next();
444       String name = (String ) entry.getKey();
445       JMethod method = (JMethod) entry.getValue();
446       JMethod beginMethod = getBeginMethod(type, name);
447       JMethod endMethod = getEndMethod(type, name);
448
449       sw.println("public void " + name + "() {");
450       sw.indent();
451       sw.println(" privateDelayTestFinish( 2000 );");
452       sw.println();
453
454       MutableBoolean isBounded = new MutableBoolean();
455       Map params = getParamMetaData(method, isBounded);
456       validateParams(method, params);
457
458       JParameter[] methodParams = method.getParameters();
459       List paramNames = new ArrayList (methodParams.length);
460       for (int i = 0; i < methodParams.length; ++i) {
461         paramNames.add(methodParams[i].getName());
462       }
463
464       List paramValues = new ArrayList (methodParams.length);
465       for (int i = 0; i < methodParams.length; ++i) {
466         paramValues.add(params.get(methodParams[i].getName()));
467       }
468
469       sw.print( "final java.util.List ranges = java.util.Arrays.asList( new com.google.gwt.junit.client.Range[] { " );
470
471       for (int i = 0; i < paramNames.size(); ++i) {
472         String paramName = (String ) paramNames.get(i);
473         sw.print( (String ) params.get(paramName) );
474         if (i != paramNames.size() - 1) {
475           sw.print( ",");
476         } else {
477           sw.println( "} );" );
478         }
479         sw.print( " " );
480       }
481
482       sw.println(
483           "final com.google.gwt.junit.client.impl.PermutationIterator permutationIt = new com.google.gwt.junit.client.impl.PermutationIterator( ranges );\n" +
484           "com.google.gwt.user.client.DeferredCommand.addCommand( new com.google.gwt.user.client.IncrementalCommand() {\n" +
485           " public boolean execute() {\n" +
486           " privateDelayTestFinish( 10000 );\n" +
487           " if ( permutationIt.hasNext() ) {\n" +
488           " com.google.gwt.junit.client.impl.PermutationIterator.Permutation permutation = (com.google.gwt.junit.client.impl.PermutationIterator.Permutation) permutationIt.next();\n"
489       );
490
491       for (int i = 0; i < methodParams.length; ++i) {
492         JParameter methodParam = methodParams[i];
493         String typeName = methodParam.getType().getQualifiedSourceName();
494         String paramName = (String ) paramNames.get(i);
495         sw.println( " " + typeName + " " + paramName + " = (" +
496                     typeName + ") permutation.getValues().get(" + i + ");");
497       }
498
499       final String setupTimingName = "__setupTiming";
500       final String testTimingName = "__testTiming";
501
502       sw.println("double " + setupTimingName + " = 0;");
503       sw.println("double " + testTimingName + " = 0;");
504
505       Statements setupBench = genBenchTarget(beginMethod, endMethod, paramNames,
506           new Statement(new MethodCall(EMPTY_FUNC, null)));
507       Statements testBench = genBenchTarget(beginMethod, endMethod, paramNames,
508           new Statement(new MethodCall(method.getName(), paramNames)));
509
510       StringBuffer recordResultsCode = new StringBuffer (
511           "com.google.gwt.junit.client.TestResults results = getTestResults();\n" +
512           "com.google.gwt.junit.client.Trial trial = new com.google.gwt.junit.client.Trial();\n" +
513           "trial.setRunTimeMillis( " + testTimingName + " - " + setupTimingName + " );\n" +
514           "java.util.Map variables = trial.getVariables();\n");
515
516       for (int i = 0; i < paramNames.size(); ++i) {
517         String paramName = (String ) paramNames.get(i);
518         recordResultsCode.append("variables.put( \"")
519             .append(paramName)
520             .append("\", ")
521             .append(paramName)
522             .append(".toString() );\n");
523       }
524
525       recordResultsCode.append("results.getTrials().add( trial )");
526       Statements recordCode = new Statement(recordResultsCode.toString());
527
528       Statements breakCode = new Statement( " permutationIt.skipCurrentRange()" );
529       setupBench = benchmark(setupBench, setupTimingName, false, null, breakCode);
530       testBench = benchmark(testBench, testTimingName, isBounded.value, recordCode, breakCode);
531
532       Statements testAndSetup = new StatementsList();
533       testAndSetup.getStatements().addAll(setupBench.getStatements());
534       testAndSetup.getStatements().addAll(testBench.getStatements());
535
536       sw.println( testAndSetup.toString() );
537
538       sw.println(
539           " return true;\n" +
540           " }\n" +
541           " privateFinishTest();\n" +
542           " return false;\n" +
543           " }\n" +
544           "} );\n"
545       );
546
547       sw.outdent();
548       sw.println("}");
549     }
550   }
551
552   /**
553    * Overrides the zero-arg test methods that don't have any
554    * overloaded/parameterized versions.
555    *
556    * TODO(tobyr) This code shares a lot of similarity with
557    * implementParameterizedTestMethods and they should probably be refactored
558    * into a single function.
559    */
560   private void implementZeroArgTestMethods() {
561     Map zeroArgMethods = getNotOverloadedTestMethods(getRequestedClass());
562     SourceWriter sw = getSourceWriter();
563     JClassType type = getRequestedClass();
564
565     for (Iterator it = zeroArgMethods.entrySet().iterator(); it.hasNext();) {
566       Map.Entry entry = (Map.Entry ) it.next();
567       String name = (String ) entry.getKey();
568       JMethod method = (JMethod) entry.getValue();
569       JMethod beginMethod = getBeginMethod(type, name);
570       JMethod endMethod = getEndMethod(type, name);
571
572       sw.println("public void " + name + "() {");
573       sw.indent();
574
575       final String setupTimingName = "__setupTiming";
576       final String testTimingName = "__testTiming";
577
578       sw.println("double " + setupTimingName + " = 0;");
579       sw.println("double " + testTimingName + " = 0;");
580
581       Statements setupBench = genBenchTarget(beginMethod, endMethod,
582           Collections.EMPTY_LIST,
583           new Statement(new MethodCall(EMPTY_FUNC, null)));
584
585       StatementsList testStatements = new StatementsList();
586       testStatements.getStatements().add(
587           new Statement(new MethodCall("super." + method.getName(), null)));
588       Statements testBench = genBenchTarget(beginMethod, endMethod,
589           Collections.EMPTY_LIST, testStatements);
590
591       String recordResultsCode =
592           "com.google.gwt.junit.client.TestResults results = getTestResults();\n" +
593           "com.google.gwt.junit.client.Trial trial = new com.google.gwt.junit.client.Trial();\n" +
594           "trial.setRunTimeMillis( " + testTimingName + " - " + setupTimingName + " );\n" +
595           "results.getTrials().add( trial )";
596
597       Statements breakCode = new Statement( " break " + ESCAPE_LOOP );
598
599       setupBench = benchmark(setupBench, setupTimingName, false, null, breakCode);
600       testBench = benchmark(testBench, testTimingName, true,
601           new Statement(recordResultsCode), breakCode);
602       ForLoop loop = (ForLoop) testBench.getStatements().get(0);
603       loop.setLabel(ESCAPE_LOOP);
604
605       sw.println(setupBench.toString());
606       sw.println(testBench.toString());
607
608       sw.outdent();
609       sw.println("}");
610     }
611   }
612
613   private void validateParams(JMethod method, Map params)
614       throws UnableToCompleteException {
615     JParameter[] methodParams = method.getParameters();
616     for (int i = 0; i < methodParams.length; ++i) {
617       JParameter methodParam = methodParams[i];
618       String paramName = methodParam.getName();
619       String paramValue = (String ) params.get(paramName);
620
621       if (paramValue == null) {
622         String msg = "Could not find the meta data attribute "
623             + BENCHMARK_PARAM_META +
624             " for the parameter " + paramName + " on method " + method
625             .getName();
626         logger.log(TreeLogger.ERROR, msg, null);
627         throw new UnableToCompleteException();
628       }
629     }
630   }
631 }
632

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags