aboutsummaryrefslogtreecommitdiff
path: root/src/com/google/caliper/Runner.java
diff options
context:
space:
mode:
Diffstat (limited to 'src/com/google/caliper/Runner.java')
-rw-r--r--src/com/google/caliper/Runner.java363
1 files changed, 98 insertions, 265 deletions
diff --git a/src/com/google/caliper/Runner.java b/src/com/google/caliper/Runner.java
index 72442db..e359df8 100644
--- a/src/com/google/caliper/Runner.java
+++ b/src/com/google/caliper/Runner.java
@@ -16,180 +16,113 @@
package com.google.caliper;
+import com.google.caliper.UserException.ExceptionFromUserCodeException;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.LinkedHashMultimap;
-import com.google.common.collect.Multimap;
-
+import com.google.common.collect.ImmutableMap.Builder;
+import com.google.common.collect.ObjectArrays;
import java.io.BufferedReader;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.*;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.UUID;
/**
* Creates, executes and reports benchmark runs.
*/
public final class Runner {
- private String suiteClassName;
- private Benchmark suite;
-
- /** Effective parameters to run in the benchmark. */
- private Multimap<String, String> parameters = LinkedHashMultimap.create();
-
- /** JVMs to run in the benchmark */
- private Set<String> userVms = new LinkedHashSet<String>();
-
- /**
- * Parameter values specified by the user on the command line. Parameters with
- * no value in this multimap will get their values from the benchmark suite.
- */
- private Multimap<String, String> userParameters = LinkedHashMultimap.create();
-
- /**
- * True if each benchmark should run in process.
- */
- private boolean inProcess;
-
- private long warmupMillis = 5000;
- private long runMillis = 5000;
+ /** Command line arguments to the process */
+ private Arguments arguments;
+ private ScenarioSelection scenarioSelection;
/**
- * Sets the named parameter to the specified value. This value will replace
- * the benchmark suite's default values for the parameter. Multiple calls to
- * this method will cause benchmarks for each value to be run.
+ * Returns the UUID of the executing host. Multiple runs by the same user on
+ * the same machine should yield the same result.
*/
- void setParameter(String name, String value) {
- userParameters.put(name, value);
- }
-
- private void prepareSuite() {
+ private String getExecutedByUuid() {
try {
- @SuppressWarnings("unchecked") // guarded by the if statement that follows
- Class<? extends Benchmark> suiteClass
- = (Class<? extends Benchmark>) Class.forName(suiteClassName);
- if (!Benchmark.class.isAssignableFrom(suiteClass)) {
- throw new ConfigurationException(suiteClass + " is not a benchmark suite.");
+ File dotCaliperRc = new File(System.getProperty("user.home"), ".caliperrc");
+ Properties properties = new Properties();
+ if (dotCaliperRc.exists()) {
+ properties.load(new FileInputStream(dotCaliperRc));
}
- Constructor<? extends Benchmark> constructor = suiteClass.getDeclaredConstructor();
- suite = constructor.newInstance();
- } catch (InvocationTargetException e) {
- throw new ExecutionException(e.getCause());
- } catch (Exception e) {
- throw new ConfigurationException(e);
- }
- }
-
- private void prepareParameters() {
- for (String key : suite.parameterNames()) {
- // first check if the user has specified values
- Collection<String> userValues = userParameters.get(key);
- if (!userValues.isEmpty()) {
- parameters.putAll(key, userValues);
- // TODO: type convert 'em to validate?
-
- } else { // otherwise use the default values from the suite
- Set<String> values = suite.parameterValues(key);
- if (values.isEmpty()) {
- throw new ConfigurationException(key + " has no values");
- }
- parameters.putAll(key, values);
+ String userUuid = properties.getProperty("userUuid");
+ if (userUuid == null) {
+ userUuid = UUID.randomUUID().toString();
+ properties.setProperty("userUuid", userUuid);
+ properties.store(new FileOutputStream(dotCaliperRc), "");
}
+
+ return userUuid;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
}
- private ImmutableSet<String> defaultVms() {
- return "Dalvik".equals(System.getProperty("java.vm.name"))
- ? ImmutableSet.of("dalvikvm")
- : ImmutableSet.of("java");
+ public void run(String... args) {
+ this.arguments = Arguments.parse(args);
+ this.scenarioSelection = new ScenarioSelection(arguments);
+ Run run = runOutOfProcess();
+ new ConsoleReport(run).displayResults();
+ postResults(run);
}
- /**
- * Returns a complete set of runs with every combination of values and
- * benchmark classes.
- */
- private List<Run> createRuns() throws Exception {
- List<RunBuilder> builders = new ArrayList<RunBuilder>();
-
- // create runs for each VMs
- Set<String> vms = userVms.isEmpty()
- ? defaultVms()
- : userVms;
- for (String vm : vms) {
- RunBuilder runBuilder = new RunBuilder();
- runBuilder.vm = vm;
- builders.add(runBuilder);
+ private void postResults(Run run) {
+ String postHost = arguments.getPostHost();
+ if ("none".equals(postHost)) {
+ return;
}
- for (Map.Entry<String, Collection<String>> parameter : parameters.asMap().entrySet()) {
- Iterator<String> values = parameter.getValue().iterator();
- if (!values.hasNext()) {
- throw new ConfigurationException("Not enough values for " + parameter);
- }
-
- String key = parameter.getKey();
-
- String firstValue = values.next();
- for (RunBuilder builder : builders) {
- builder.parameters.put(key, firstValue);
+ try {
+ URL url = new URL(postHost + run.getExecutedByUuid() + "/" + run.getBenchmarkName());
+ HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
+ urlConnection.setDoOutput(true);
+ Xml.runToXml(run, urlConnection.getOutputStream());
+ if (urlConnection.getResponseCode() == 200) {
+ System.out.println("");
+ System.out.println("View current and previous benchmark results online:");
+ System.out.println(" " + url);
+ return;
}
- // multiply the size of the specs by the number of alternate values
- int size = builders.size();
- while (values.hasNext()) {
- String alternate = values.next();
- for (int s = 0; s < size; s++) {
- RunBuilder copy = builders.get(s).copy();
- copy.parameters.put(key, alternate);
- builders.add(copy);
- }
+ System.out.println("Posting to " + postHost + " failed: "
+ + urlConnection.getResponseMessage());
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(urlConnection.getInputStream()));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ System.out.println(line);
}
- }
-
- List<Run> result = new ArrayList<Run>();
- for (RunBuilder builder : builders) {
- result.add(builder.build());
- }
-
- return result;
- }
-
- static class RunBuilder {
- Map<String, String> parameters = new LinkedHashMap<String, String>();
- String vm;
-
- RunBuilder copy() {
- RunBuilder result = new RunBuilder();
- result.parameters.putAll(parameters);
- result.vm = vm;
- return result;
- }
-
- public Run build() {
- return new Run(parameters, vm);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
}
- private double executeForked(Run run) {
+ private double executeForked(Scenario scenario) {
ProcessBuilder builder = new ProcessBuilder();
List<String> command = builder.command();
- command.addAll(Arrays.asList(run.getVm().split("\\s+")));
+ command.addAll(Arrays.asList(scenario.getVariables().get(Scenario.VM_KEY).split("\\s+")));
command.add("-cp");
command.add(System.getProperty("java.class.path"));
- command.add(Runner.class.getName());
+ command.add(InProcessRunner.class.getName());
command.add("--warmupMillis");
- command.add(String.valueOf(warmupMillis));
+ command.add(String.valueOf(arguments.getWarmupMillis()));
command.add("--runMillis");
- command.add(String.valueOf(runMillis));
- command.add("--inProcess");
- for (Map.Entry<String, String> entry : run.getParameters().entrySet()) {
+ command.add(String.valueOf(arguments.getRunMillis()));
+ for (Entry<String, String> entry : scenario.getParameters().entrySet()) {
command.add("-D" + entry.getKey() + "=" + entry.getValue());
}
- command.add(suiteClassName);
+ command.add(arguments.getSuiteClassName());
BufferedReader reader = null;
try {
@@ -202,7 +135,7 @@ public final class Runner {
Double nanosPerTrial = null;
try {
nanosPerTrial = Double.valueOf(firstLine);
- } catch (NumberFormatException e) {
+ } catch (NumberFormatException ignore) {
}
String anotherLine = reader.readLine();
@@ -229,163 +162,63 @@ public final class Runner {
}
}
- private Result runOutOfProcess() {
- ImmutableMap.Builder<Run, Double> resultsBuilder = ImmutableMap.builder();
+ // TODO: check if this is platform-independent
+ @SuppressWarnings("HardcodedLineSeparator")
+ private static final String RETURN = "\r";
+
+ private Run runOutOfProcess() {
+ String executedByUuid = getExecutedByUuid();
+ Date executedDate = new Date();
+ Builder<Scenario, Double> resultsBuilder = ImmutableMap.builder();
try {
- List<Run> runs = createRuns();
+ List<Scenario> scenarios = scenarioSelection.select();
int i = 0;
- for (Run run : runs) {
- beforeRun(i++, runs.size(), run);
- double nanosPerTrial = executeForked(run);
- afterRun(nanosPerTrial);
- resultsBuilder.put(run, nanosPerTrial);
+ for (Scenario scenario : scenarios) {
+ beforeMeasurement(i++, scenarios.size(), scenario);
+ double nanosPerTrial = executeForked(scenario);
+ afterMeasurement(nanosPerTrial);
+ resultsBuilder.put(scenario, nanosPerTrial);
}
// blat out our progress bar
- System.out.print("\r");
+ System.out.print(RETURN);
for (int j = 0; j < 80; j++) {
System.out.print(" ");
}
- System.out.print("\r");
+ System.out.print(RETURN);
- return new Result(resultsBuilder.build());
+ return new Run(resultsBuilder.build(), arguments.getSuiteClassName(), executedByUuid, executedDate);
} catch (Exception e) {
- throw new ExecutionException(e);
+ throw new ExceptionFromUserCodeException(e);
}
}
- private void beforeRun(int index, int total, Run run) {
+ private void beforeMeasurement(int index, int total, Scenario scenario) {
double percentDone = (double) index / total;
int runStringLength = 63; // so the total line length is 80
- String runString = String.valueOf(run);
+ String runString = String.valueOf(scenario);
if (runString.length() > runStringLength) {
runString = runString.substring(0, runStringLength);
}
- System.out.printf("\r%2.0f%% %-" + runStringLength + "s",
+ System.out.printf(RETURN + "%2.0f%% %-" + runStringLength + "s",
percentDone * 100, runString);
}
- private void afterRun(double nanosPerTrial) {
+ private void afterMeasurement(double nanosPerTrial) {
System.out.printf(" %10.0fns", nanosPerTrial);
}
- private void runInProcess() {
+ public static void main(String... args) {
try {
- Caliper caliper = new Caliper(warmupMillis, runMillis);
-
- for (Run run : createRuns()) {
- double result;
- TimedRunnable timedRunnable = suite.createBenchmark(run.getParameters());
- double warmupNanosPerTrial = caliper.warmUp(timedRunnable);
- result = caliper.run(timedRunnable, warmupNanosPerTrial);
- double nanosPerTrial = result;
- System.out.println(nanosPerTrial);
- }
- } catch (Exception e) {
- throw new ExecutionException(e);
- }
- }
-
- private boolean parseArgs(String[] args) throws Exception {
- for (int i = 0; i < args.length; i++) {
- if ("--help".equals(args[i])) {
- return false;
-
- } else if ("--inProcess".equals(args[i])) {
- inProcess = true;
-
- } else if (args[i].startsWith("-D")) {
- int equalsSign = args[i].indexOf('=');
- if (equalsSign == -1) {
- System.out.println("Malformed parameter " + args[i]);
- return false;
- }
- String name = args[i].substring(2, equalsSign);
- String value = args[i].substring(equalsSign + 1);
- setParameter(name, value);
-
- } else if ("--warmupMillis".equals(args[i])) {
- warmupMillis = Long.parseLong(args[++i]);
-
- } else if ("--runMillis".equals(args[i])) {
- runMillis = Long.parseLong(args[++i]);
-
- } else if ("--vm".equals(args[i])) {
- userVms.add(args[++i]);
-
- } else if (args[i].startsWith("-")) {
- System.out.println("Unrecognized option: " + args[i]);
- return false;
-
- } else {
- if (suiteClassName != null) {
- System.out.println("Too many benchmark classes!");
- return false;
- }
- suiteClassName = args[i];
- }
- }
-
- if (inProcess && !userVms.isEmpty()) {
- System.out.println("Cannot customize VM when running in process");
- return false;
- }
-
- if (suiteClassName == null) {
- System.out.println("No benchmark class provided.");
- return false;
- }
-
- return true;
- }
-
- private void printUsage() {
- System.out.println("Usage: Runner [OPTIONS...] <benchmark>");
- System.out.println();
- System.out.println(" <benchmark>: a benchmark class or suite");
- System.out.println();
- System.out.println("OPTIONS");
- System.out.println();
- System.out.println(" --D<param>=<value>: fix a benchmark parameter to a given value.");
- System.out.println(" When multiple values for the same parameter are given (via");
- System.out.println(" multiple --Dx=y args), all supplied values are used.");
- System.out.println();
- System.out.println(" --inProcess: run the benchmark in the same JVM rather than spawning");
- System.out.println(" another with the same classpath. By default each benchmark is");
- System.out.println(" run in a separate VM");
- System.out.println();
- System.out.println(" --warmupMillis <millis>: duration to warmup each benchmark");
- System.out.println();
- System.out.println(" --runMillis <millis>: duration to execute each benchmark");
- System.out.println();
- System.out.println(" --vm <vm>: executable to test benchmark on");
-
- // adding new options? don't forget to update executeForked()
- }
-
- public static void main(String... args) throws Exception { // TODO: cleaner error reporting
- Runner runner = new Runner();
- if (!runner.parseArgs(args)) {
- runner.printUsage();
- return;
+ new Runner().run(args);
+ } catch (UserException e) {
+ e.display();
+ System.exit(1);
}
-
- runner.prepareSuite();
- runner.prepareParameters();
- if (runner.inProcess) {
- runner.runInProcess();
- return;
- }
-
- Result result = runner.runOutOfProcess();
- new ConsoleReport(result).displayResults();
}
- public static void main(Class<? extends Benchmark> suite, String... args) throws Exception {
- String[] argsWithSuiteName = new String[args.length + 1];
- System.arraycopy(args, 0, argsWithSuiteName, 0, args.length);
- argsWithSuiteName[args.length] = suite.getName();
- main(argsWithSuiteName);
+ public static void main(Class<? extends Benchmark> suite, String... args) {
+ main(ObjectArrays.concat(args, suite.getName()));
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-30 05:41:33 +0000