-- cgit v1.2.3 From 7fc0b45feb5ac6191547bb0a175a718cb41ec4f9 Mon Sep 17 00:00:00 2001 From: Paul Duffin Date: Tue, 10 Nov 2015 17:45:15 +0000 Subject: Added version 0.5-rc1 Change-Id: Ic21eda3c659eadf3a4229ae5618e9ce1e96b4aa7 --- Android.mk | 43 ++ COPYING | 202 ++++++++++ README | 8 + caliper/pom.xml | 134 +++++++ .../com/google/caliper/AllocationMeasurer.java | 165 ++++++++ .../main/java/com/google/caliper/Arguments.java | 420 ++++++++++++++++++++ .../main/java/com/google/caliper/Benchmark.java | 55 +++ .../main/java/com/google/caliper/CaliperRc.java | 63 +++ .../com/google/caliper/ConfigurationException.java | 33 ++ .../com/google/caliper/ConfiguredBenchmark.java | 67 ++++ .../java/com/google/caliper/ConsoleReport.java | 427 ++++++++++++++++++++ .../com/google/caliper/CountingPrintStream.java | 77 ++++ .../src/main/java/com/google/caliper/DalvikVm.java | 62 +++ .../java/com/google/caliper/DebugMeasurer.java | 42 ++ .../main/java/com/google/caliper/Environment.java | 63 +++ .../java/com/google/caliper/EnvironmentGetter.java | 178 +++++++++ .../java/com/google/caliper/InProcessRunner.java | 117 ++++++ .../caliper/InstancesAllocationMeasurer.java | 34 ++ caliper/src/main/java/com/google/caliper/Json.java | 226 +++++++++++ .../main/java/com/google/caliper/LogConstants.java | 43 ++ .../main/java/com/google/caliper/Measurement.java | 93 +++++ .../java/com/google/caliper/MeasurementSet.java | 264 +++++++++++++ .../java/com/google/caliper/MeasurementType.java | 24 ++ .../src/main/java/com/google/caliper/Measurer.java | 44 +++ .../google/caliper/MemoryAllocationMeasurer.java | 34 ++ .../src/main/java/com/google/caliper/Param.java | 61 +++ .../main/java/com/google/caliper/Parameter.java | 202 ++++++++++ .../src/main/java/com/google/caliper/Result.java | 56 +++ .../java/com/google/caliper/ResultsReader.java | 64 +++ caliper/src/main/java/com/google/caliper/Run.java | 93 +++++ .../src/main/java/com/google/caliper/Runner.java | 438 +++++++++++++++++++++ .../src/main/java/com/google/caliper/Scenario.java | 80 ++++ .../java/com/google/caliper/ScenarioResult.java | 88 +++++ .../java/com/google/caliper/ScenarioSelection.java | 253 ++++++++++++ .../java/com/google/caliper/SimpleBenchmark.java | 227 +++++++++++ .../main/java/com/google/caliper/StandardVm.java | 46 +++ .../main/java/com/google/caliper/TimeMeasurer.java | 180 +++++++++ .../java/com/google/caliper/TypeConverter.java | 65 +++ .../java/com/google/caliper/UploadResults.java | 28 ++ .../java/com/google/caliper/UserException.java | 196 +++++++++ caliper/src/main/java/com/google/caliper/Vm.java | 46 +++ .../main/java/com/google/caliper/VmFactory.java | 47 +++ caliper/src/main/java/com/google/caliper/Xml.java | 160 ++++++++ .../src/main/java/com/google/caliper/XmlUtils.java | 49 +++ .../com/google/caliper/util/InterleavedReader.java | 128 ++++++ .../com/google/caliper/util/LinearTranslation.java | 48 +++ caliper/src/main/resources/CaliperCore.gwt.xml | 15 + .../src/test/java/com/google/caliper/AllTests.java | 33 ++ .../test/java/com/google/caliper/CaliperTest.java | 93 +++++ .../src/test/java/com/google/caliper/JsonTest.java | 67 ++++ .../com/google/caliper/MeasurementSetTest.java | 216 ++++++++++ .../java/com/google/caliper/ParameterTest.java | 71 ++++ .../com/google/caliper/WarmupOverflowTest.java | 81 ++++ examples/pom.xml | 88 +++++ .../src/main/java/examples/ArraySortBenchmark.java | 102 +++++ .../src/main/java/examples/BitSetBenchmark.java | 186 +++++++++ .../src/main/java/examples/CharacterBenchmark.java | 300 ++++++++++++++ .../java/examples/CompressionSizeBenchmark.java | 90 +++++ .../src/main/java/examples/ContainsBenchmark.java | 78 ++++ .../src/main/java/examples/CopyArrayBenchmark.java | 356 +++++++++++++++++ examples/src/main/java/examples/DemoBenchmark.java | 74 ++++ .../java/examples/DoubleToStringBenchmark.java | 107 +++++ .../java/examples/DoubleToStringBenchmark2.java | 111 ++++++ .../java/examples/EnumSetContainsBenchmark.java | 92 +++++ .../java/examples/ExpensiveObjectsBenchmark.java | 76 ++++ .../src/main/java/examples/FormatterBenchmark.java | 76 ++++ .../src/main/java/examples/IntModBenchmark.java | 94 +++++ .../main/java/examples/ListIterationBenchmark.java | 73 ++++ .../java/examples/LoopingBackwardsBenchmark.java | 52 +++ .../examples/MessageDigestCreationBenchmark.java | 44 +++ .../main/java/examples/StringBuilderBenchmark.java | 132 +++++++ lib/gson-1.7.1.jar | Bin 0 -> 173590 bytes lib/java-allocation-instrumenter-2.0.jar | Bin 0 -> 20567 bytes scripts/caliper | 9 + tutorial/Tutorial.java | 187 +++++++++ 75 files changed, 8376 insertions(+) create mode 100644 Android.mk create mode 100644 COPYING create mode 100644 README create mode 100644 caliper/pom.xml create mode 100644 caliper/src/main/java/com/google/caliper/AllocationMeasurer.java create mode 100644 caliper/src/main/java/com/google/caliper/Arguments.java create mode 100644 caliper/src/main/java/com/google/caliper/Benchmark.java create mode 100644 caliper/src/main/java/com/google/caliper/CaliperRc.java create mode 100644 caliper/src/main/java/com/google/caliper/ConfigurationException.java create mode 100644 caliper/src/main/java/com/google/caliper/ConfiguredBenchmark.java create mode 100644 caliper/src/main/java/com/google/caliper/ConsoleReport.java create mode 100644 caliper/src/main/java/com/google/caliper/CountingPrintStream.java create mode 100644 caliper/src/main/java/com/google/caliper/DalvikVm.java create mode 100644 caliper/src/main/java/com/google/caliper/DebugMeasurer.java create mode 100644 caliper/src/main/java/com/google/caliper/Environment.java create mode 100644 caliper/src/main/java/com/google/caliper/EnvironmentGetter.java create mode 100644 caliper/src/main/java/com/google/caliper/InProcessRunner.java create mode 100644 caliper/src/main/java/com/google/caliper/InstancesAllocationMeasurer.java create mode 100644 caliper/src/main/java/com/google/caliper/Json.java create mode 100644 caliper/src/main/java/com/google/caliper/LogConstants.java create mode 100644 caliper/src/main/java/com/google/caliper/Measurement.java create mode 100644 caliper/src/main/java/com/google/caliper/MeasurementSet.java create mode 100644 caliper/src/main/java/com/google/caliper/MeasurementType.java create mode 100644 caliper/src/main/java/com/google/caliper/Measurer.java create mode 100644 caliper/src/main/java/com/google/caliper/MemoryAllocationMeasurer.java create mode 100644 caliper/src/main/java/com/google/caliper/Param.java create mode 100644 caliper/src/main/java/com/google/caliper/Parameter.java create mode 100644 caliper/src/main/java/com/google/caliper/Result.java create mode 100644 caliper/src/main/java/com/google/caliper/ResultsReader.java create mode 100644 caliper/src/main/java/com/google/caliper/Run.java create mode 100644 caliper/src/main/java/com/google/caliper/Runner.java create mode 100644 caliper/src/main/java/com/google/caliper/Scenario.java create mode 100644 caliper/src/main/java/com/google/caliper/ScenarioResult.java create mode 100644 caliper/src/main/java/com/google/caliper/ScenarioSelection.java create mode 100644 caliper/src/main/java/com/google/caliper/SimpleBenchmark.java create mode 100644 caliper/src/main/java/com/google/caliper/StandardVm.java create mode 100644 caliper/src/main/java/com/google/caliper/TimeMeasurer.java create mode 100644 caliper/src/main/java/com/google/caliper/TypeConverter.java create mode 100644 caliper/src/main/java/com/google/caliper/UploadResults.java create mode 100644 caliper/src/main/java/com/google/caliper/UserException.java create mode 100644 caliper/src/main/java/com/google/caliper/Vm.java create mode 100644 caliper/src/main/java/com/google/caliper/VmFactory.java create mode 100644 caliper/src/main/java/com/google/caliper/Xml.java create mode 100644 caliper/src/main/java/com/google/caliper/XmlUtils.java create mode 100644 caliper/src/main/java/com/google/caliper/util/InterleavedReader.java create mode 100644 caliper/src/main/java/com/google/caliper/util/LinearTranslation.java create mode 100644 caliper/src/main/resources/CaliperCore.gwt.xml create mode 100644 caliper/src/test/java/com/google/caliper/AllTests.java create mode 100644 caliper/src/test/java/com/google/caliper/CaliperTest.java create mode 100644 caliper/src/test/java/com/google/caliper/JsonTest.java create mode 100644 caliper/src/test/java/com/google/caliper/MeasurementSetTest.java create mode 100644 caliper/src/test/java/com/google/caliper/ParameterTest.java create mode 100644 caliper/src/test/java/com/google/caliper/WarmupOverflowTest.java create mode 100644 examples/pom.xml create mode 100644 examples/src/main/java/examples/ArraySortBenchmark.java create mode 100644 examples/src/main/java/examples/BitSetBenchmark.java create mode 100644 examples/src/main/java/examples/CharacterBenchmark.java create mode 100644 examples/src/main/java/examples/CompressionSizeBenchmark.java create mode 100644 examples/src/main/java/examples/ContainsBenchmark.java create mode 100644 examples/src/main/java/examples/CopyArrayBenchmark.java create mode 100644 examples/src/main/java/examples/DemoBenchmark.java create mode 100644 examples/src/main/java/examples/DoubleToStringBenchmark.java create mode 100644 examples/src/main/java/examples/DoubleToStringBenchmark2.java create mode 100644 examples/src/main/java/examples/EnumSetContainsBenchmark.java create mode 100644 examples/src/main/java/examples/ExpensiveObjectsBenchmark.java create mode 100644 examples/src/main/java/examples/FormatterBenchmark.java create mode 100644 examples/src/main/java/examples/IntModBenchmark.java create mode 100644 examples/src/main/java/examples/ListIterationBenchmark.java create mode 100644 examples/src/main/java/examples/LoopingBackwardsBenchmark.java create mode 100644 examples/src/main/java/examples/MessageDigestCreationBenchmark.java create mode 100644 examples/src/main/java/examples/StringBuilderBenchmark.java create mode 100644 lib/gson-1.7.1.jar create mode 100644 lib/java-allocation-instrumenter-2.0.jar create mode 100644 scripts/caliper create mode 100644 tutorial/Tutorial.java diff --git a/Android.mk b/Android.mk new file mode 100644 index 0000000..c747764 --- /dev/null +++ b/Android.mk @@ -0,0 +1,43 @@ +# Copyright (C) 2014 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOCAL_PATH := $(call my-dir) + +# build caliper jar +# ============================================================ + +include $(CLEAR_VARS) + +LOCAL_MODULE := caliper-target +LOCAL_MODULE_TAGS := optional +LOCAL_MODULE_CLASS := JAVA_LIBRARIES +LOCAL_SRC_FILES := $(call all-java-files-under, caliper/src/main/java/) +LOCAL_JAVA_RESOURCE_DIRS := caliper/src/main/resources + +LOCAL_STATIC_JAVA_LIBRARIES := \ + caliper-gson \ + caliper-java-allocation-instrumenter \ + guava + +include $(BUILD_STATIC_JAVA_LIBRARY) + +# Build dependencies. +# ============================================================ +include $(CLEAR_VARS) + +LOCAL_PREBUILT_STATIC_JAVA_LIBRARIES := \ + caliper-gson:lib/gson-1.7.1$(COMMON_JAVA_PACKAGE_SUFFIX) \ + caliper-java-allocation-instrumenter:lib/java-allocation-instrumenter-2.0$(COMMON_JAVA_PACKAGE_SUFFIX) + +include $(BUILD_MULTI_PREBUILT) diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README b/README new file mode 100644 index 0000000..c54683c --- /dev/null +++ b/README @@ -0,0 +1,8 @@ +To build this project with Maven: + +1. cd caliper +mvn eclipse:configure-workspace eclipse:eclipse install + +2. To build examples +cd examples +mvn eclipse:configure-workspace eclipse:eclipse install diff --git a/caliper/pom.xml b/caliper/pom.xml new file mode 100644 index 0000000..befd583 --- /dev/null +++ b/caliper/pom.xml @@ -0,0 +1,134 @@ + + 4.0.0 + com.google.caliper + caliper + jar + 0.5-rc1 + 2009 + caliper + + org.sonatype.oss + oss-parent + 7 + + http://code.google.com/p/caliper/ + Caliper: Microbenchmarking Framework for Java + + UTF-8 + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + scm:git:http://code.google.com/p/caliper/ + scm:git:http://code.google.com/p/caliper/ + http://code.google.com/p/caliper/source/browse + + + Google Code Issue Tracking + http://code.google.com/p/caliper/issues/list + + + Google, Inc. + http://www.google.com + + + + com.google.code.findbugs + jsr305 + 1.3.9 + + + com.google.code.gson + gson + 1.7.1 + + + com.google.guava + guava + 11.0.1 + compile + + + com.google.code.java-allocation-instrumenter + java-allocation-instrumenter + 2.0 + + + junit + junit + 3.8.2 + test + + + + package + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 1.6 + 1.6 + + + + org.apache.maven.plugins + maven-eclipse-plugin + 2.8 + + true + true + ../eclipse-ws/ + + + + org.apache.maven.plugins + maven-source-plugin + 2.1.2 + + + attach-sources + verify + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.7 + + + generate-javadocs + site + + javadoc + + + + + + http://download.oracle.com/javase/1.5.0/docs/api/ + + true + public + + + + + + + Jesse Wilson + Google Inc. + + + diff --git a/caliper/src/main/java/com/google/caliper/AllocationMeasurer.java b/caliper/src/main/java/com/google/caliper/AllocationMeasurer.java new file mode 100644 index 0000000..2ec3488 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/AllocationMeasurer.java @@ -0,0 +1,165 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.caliper.UserException.NonConstantMemoryUsage; +import com.google.common.base.Supplier; +import com.google.monitoring.runtime.instrumentation.AllocationRecorder; +import com.google.monitoring.runtime.instrumentation.Sampler; + +public abstract class AllocationMeasurer extends Measurer { + + protected static final int ALLOCATION_DISPLAY_THRESHOLD = 50; + + private boolean log; + private long tempAllocationCount; + private long allocationsToIgnore; + private long numberOfAllocations; + private long allocationCount; + private long outOfThreadAllocationCount; + private boolean recordAllocations; + protected String type; + + protected AllocationMeasurer() { + log = false; + allocationsToIgnore = 0; + numberOfAllocations = 0; + allocationCount = 0; + outOfThreadAllocationCount = 0; + recordAllocations = false; + + final Thread allocatingThread = Thread.currentThread(); + AllocationRecorder.addSampler(new Sampler() { + // allocated {@code newObj} of type {@code desc}, whose size is {@code size}. + // if this was not an array, {@code count} is -1. If it was array, {@code count} is the + // size of the array. + @Override public void sampleAllocation(int count, String desc, Object newObj, long size) { + if (recordAllocations) { + if (Thread.currentThread().equals(allocatingThread)) { + if (log) { + logAllocation(count, desc, size); + } else if (numberOfAllocations == 0) { + log("see first run for list of allocations"); + } + allocationCount = incrementAllocationCount(allocationCount, count, size); + tempAllocationCount++; + numberOfAllocations++; + } else { + outOfThreadAllocationCount = incrementAllocationCount(outOfThreadAllocationCount, count, size); + numberOfAllocations++; + } + } + } + }); + } + + protected abstract long incrementAllocationCount(long orig, int count, long size); + + private void logAllocation(int count, String desc, long size) { + if (numberOfAllocations >= allocationsToIgnore) { + if (numberOfAllocations < ALLOCATION_DISPLAY_THRESHOLD + allocationsToIgnore) { + log("allocating " + desc + (count == -1 ? "" : " array with " + count + " elements") + + " with size " + size + " bytes"); + } else if (numberOfAllocations == ALLOCATION_DISPLAY_THRESHOLD + allocationsToIgnore) { + log("...more allocations..."); + } + } + } + + @Override public MeasurementSet run(Supplier testSupplier) throws Exception { + + // warm up, for some reason the very first time anything is measured, it will have a few more + // allocations. + measureAllocations(testSupplier.get(), 1, 0); + + // The "one" case serves as a base line. There may be caching, lazy loading, etc going on here. + tempAllocationCount = 0; // count the number of times the sampler is called in one rep + long one = measureAllocationsTotal(testSupplier.get(), 1); + long oneAllocations = tempAllocationCount; + + // we expect that the delta between any two consecutive reps will be constant + tempAllocationCount = 0; // count the number of times the sampler is called in two reps + long two = measureAllocationsTotal(testSupplier.get(), 2); + long twoAllocations = tempAllocationCount; + long expectedDiff = two - one; + // there is some overhead on the first call that we can ignore for the purposes of measurement + long unitsToIgnore = one - expectedDiff; + allocationsToIgnore = 2 * oneAllocations - twoAllocations; + log("ignoring " + allocationsToIgnore + " allocation(s) per measurement as overhead"); + + Measurement[] allocationMeasurements = new Measurement[4]; + log = true; + allocationMeasurements[0] = measureAllocations(testSupplier.get(), 1, unitsToIgnore); + log = false; + for (int i = 1; i < allocationMeasurements.length; i++) { + allocationMeasurements[i] = + measureAllocations(testSupplier.get(), i + 1, unitsToIgnore); + if (Math.round(allocationMeasurements[i].getRaw()) != expectedDiff) { + throw new NonConstantMemoryUsage(); + } + } + + // The above logic guarantees that all the measurements are equal, so we only need to return a + // single measurement. + allocationsToIgnore = 0; + return new MeasurementSet(allocationMeasurements[0]); + } + + private Measurement measureAllocations(ConfiguredBenchmark benchmark, int reps, long toIgnore) + throws Exception { + prepareForTest(); + log(LogConstants.MEASURED_SECTION_STARTING); + resetAllocations(); + recordAllocations = true; + benchmark.run(reps); + recordAllocations = false; + log(LogConstants.MEASURED_SECTION_DONE); + long allocations = (allocationCount - toIgnore) / reps; + long outOfThreadAllocations = outOfThreadAllocationCount; + log(allocations + " " + type + "(s) allocated per rep"); + log(outOfThreadAllocations + " out of thread " + type + "(s) allocated in " + reps + " reps"); + benchmark.close(); + return getMeasurement(benchmark, allocations); + } + + protected abstract Measurement getMeasurement(ConfiguredBenchmark benchmark, long allocations); + + private long measureAllocationsTotal(ConfiguredBenchmark benchmark, int reps) + throws Exception { + prepareForTest(); + log(LogConstants.MEASURED_SECTION_STARTING); + resetAllocations(); + recordAllocations = true; + benchmark.run(reps); + recordAllocations = false; + log(LogConstants.MEASURED_SECTION_DONE); + long allocations = allocationCount; + long outOfThreadAllocations = outOfThreadAllocationCount; + log(allocations + " " + type + "(s) allocated in " + reps + " reps"); + if (outOfThreadAllocations > 0) { + log(outOfThreadAllocations + " out of thread " + type + "(s) allocated in " + reps + " reps"); + } + benchmark.close(); + return allocations; + } + + private void resetAllocations() { + allocationCount = 0; + outOfThreadAllocationCount = 0; + numberOfAllocations = 0; + } +} diff --git a/caliper/src/main/java/com/google/caliper/Arguments.java b/caliper/src/main/java/com/google/caliper/Arguments.java new file mode 100644 index 0000000..cceb079 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Arguments.java @@ -0,0 +1,420 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.caliper.UserException.DisplayUsageException; +import com.google.caliper.UserException.IncompatibleArgumentsException; +import com.google.caliper.UserException.InvalidParameterValueException; +import com.google.caliper.UserException.MultipleBenchmarkClassesException; +import com.google.caliper.UserException.NoBenchmarkClassException; +import com.google.caliper.UserException.UnrecognizedOptionException; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; + +import java.io.File; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * Parse command line arguments for the runner and in-process runner. + */ +public final class Arguments { + private String suiteClassName; + + /** JVMs to run in the benchmark */ + private final Set userVms = Sets.newLinkedHashSet(); + + /** + * Parameter values specified by the user on the command line. Parameters with + * no value in this multimap will get their values from the benchmark suite. + */ + private final Multimap userParameters = LinkedHashMultimap.create(); + + /** + * VM parameters like {memory=[-Xmx64,-Xmx128],jit=[-client,-server]} + */ + private final Multimap vmParameters = LinkedHashMultimap.create(); + + private int trials = 1; + private long warmupMillis = 3000; + private long runMillis = 1000; + private String timeUnit = null; + private String instanceUnit = null; + private String memoryUnit = null; + private File saveResultsFile = null; + private File uploadResultsFile = null; + private boolean captureVmLog = false; + private boolean printScore = false; + private boolean measureMemory = false; + private boolean debug = false; + private int debugReps = defaultDebugReps; + private MeasurementType measurementType; + private MeasurementType primaryMeasurementType; + + /** + * A signal that indicates a JSON object interleaved with the process output. + * Should be short, but unlikely to show up in the processes natural output. + */ + private String marker = "//ZxJ/"; + + private static final String defaultDelimiter = ","; + private static final int defaultDebugReps = 1000; + + public String getSuiteClassName() { + return suiteClassName; + } + + public Set getUserVms() { + return userVms; + } + + public int getTrials() { + return trials; + } + + public Multimap getVmParameters() { + return vmParameters; + } + + public Multimap getUserParameters() { + return userParameters; + } + + public long getWarmupMillis() { + return warmupMillis; + } + + public long getRunMillis() { + return runMillis; + } + + public String getTimeUnit() { + return timeUnit; + } + + public String getInstanceUnit() { + return instanceUnit; + } + + public String getMemoryUnit() { + return memoryUnit; + } + + public File getSaveResultsFile() { + return saveResultsFile; + } + + public File getUploadResultsFile() { + return uploadResultsFile; + } + + public boolean getCaptureVmLog() { + return captureVmLog; + } + + public boolean printScore() { + return printScore; + } + + public boolean getMeasureMemory() { + return measureMemory; + } + + public MeasurementType getMeasurementType() { + return measurementType; + } + + public MeasurementType getPrimaryMeasurementType() { + return primaryMeasurementType; + } + + public boolean getDebug() { + return debug; + } + + public int getDebugReps() { + return debugReps; + } + + public String getMarker() { + return marker; + } + + public static Arguments parse(String[] argsArray) { + Arguments result = new Arguments(); + + Iterator args = Iterators.forArray(argsArray); + String delimiter = defaultDelimiter; + Map userParameterStrings = Maps.newLinkedHashMap(); + Map vmParameterStrings = Maps.newLinkedHashMap(); + String vmString = null; + boolean standardRun = false; + while (args.hasNext()) { + String arg = args.next(); + + if ("--help".equals(arg)) { + throw new DisplayUsageException(); + } + + if (arg.startsWith("-D") || arg.startsWith("-J")) { + + /* + * Handle user parameters (-D) and VM parameters (-J) of these forms: + * + * -Dlength=100 + * -Jmemory=-Xmx1024M + * -Dlength=100,200 + * -Jmemory=-Xmx1024M,-Xmx2048M + * -Dlength 100 + * -Jmemory -Xmx1024M + * -Dlength 100,200 + * -Jmemory -Xmx1024M,-Xmx2048M + */ + + String name; + String value; + int equalsSign = arg.indexOf('='); + if (equalsSign == -1) { + name = arg.substring(2); + value = args.next(); + } else { + name = arg.substring(2, equalsSign); + value = arg.substring(equalsSign + 1); + } + + String previousValue; + if (arg.startsWith("-D")) { + previousValue = userParameterStrings.put(name, value); + } else { + previousValue = vmParameterStrings.put(name, value); + } + if (previousValue != null) { + throw new UserException.DuplicateParameterException(arg); + } + standardRun = true; + // TODO: move warmup/run to caliperrc + } else if ("--captureVmLog".equals(arg)) { + result.captureVmLog = true; + standardRun = true; + } else if ("--warmupMillis".equals(arg)) { + result.warmupMillis = Long.parseLong(args.next()); + standardRun = true; + } else if ("--runMillis".equals(arg)) { + result.runMillis = Long.parseLong(args.next()); + standardRun = true; + } else if ("--trials".equals(arg)) { + String value = args.next(); + try { + result.trials = Integer.parseInt(value); + if (result.trials < 1) { + throw new UserException.InvalidTrialsException(value); + } + } catch (NumberFormatException e) { + throw new UserException.InvalidTrialsException(value); + } + standardRun = true; + } else if ("--vm".equals(arg)) { + if (vmString != null) { + throw new UserException.DuplicateParameterException(arg); + } + vmString = args.next(); + standardRun = true; + } else if ("--delimiter".equals(arg)) { + delimiter = args.next(); + standardRun = true; + } else if ("--timeUnit".equals(arg)) { + result.timeUnit = args.next(); + standardRun = true; + } else if ("--instanceUnit".equals(arg)) { + result.instanceUnit = args.next(); + standardRun = true; + } else if ("--memoryUnit".equals(arg)) { + result.memoryUnit = args.next(); + standardRun = true; + } else if ("--saveResults".equals(arg) || "--xmlSave".equals(arg)) { + // TODO: unsupport legacy --xmlSave + result.saveResultsFile = new File(args.next()); + standardRun = true; + } else if ("--uploadResults".equals(arg)) { + result.uploadResultsFile = new File(args.next()); + } else if ("--printScore".equals(arg)) { + result.printScore = true; + standardRun = true; + } else if ("--measureMemory".equals(arg)) { + result.measureMemory = true; + standardRun = true; + } else if ("--debug".equals(arg)) { + result.debug = true; + } else if ("--debug-reps".equals(arg)) { + String value = args.next(); + try { + result.debugReps = Integer.parseInt(value); + if (result.debugReps < 1) { + throw new UserException.InvalidDebugRepsException(value); + } + } catch (NumberFormatException e) { + throw new UserException.InvalidDebugRepsException(value); + } + } else if ("--marker".equals(arg)) { + result.marker = args.next(); + } else if ("--measurementType".equals(arg)) { + String measurementType = args.next(); + try { + result.measurementType = MeasurementType.valueOf(measurementType); + } catch (Exception e) { + throw new InvalidParameterValueException(arg, measurementType); + } + standardRun = true; + } else if ("--primaryMeasurementType".equals(arg)) { + String measurementType = args.next().toUpperCase(); + try { + result.primaryMeasurementType = MeasurementType.valueOf(measurementType); + } catch (Exception e) { + throw new InvalidParameterValueException(arg, measurementType); + } + standardRun = true; + } else if (arg.startsWith("-")) { + throw new UnrecognizedOptionException(arg); + + } else { + if (result.suiteClassName != null) { + throw new MultipleBenchmarkClassesException(result.suiteClassName, arg); + } + result.suiteClassName = arg; + } + } + + Splitter delimiterSplitter = Splitter.on(delimiter); + + if (vmString != null) { + Iterables.addAll(result.userVms, delimiterSplitter.split(vmString)); + } + + Set duplicates = Sets.intersection( + userParameterStrings.keySet(), vmParameterStrings.keySet()); + if (!duplicates.isEmpty()) { + throw new UserException.DuplicateParameterException(duplicates); + } + + for (Map.Entry entry : userParameterStrings.entrySet()) { + result.userParameters.putAll(entry.getKey(), delimiterSplitter.split(entry.getValue())); + } + for (Map.Entry entry : vmParameterStrings.entrySet()) { + result.vmParameters.putAll(entry.getKey(), delimiterSplitter.split(entry.getValue())); + } + + if (standardRun && result.uploadResultsFile != null) { + throw new IncompatibleArgumentsException("--uploadResults"); + } + + if (result.suiteClassName == null && result.uploadResultsFile == null) { + throw new NoBenchmarkClassException(); + } + + if (result.primaryMeasurementType != null + && result.primaryMeasurementType != MeasurementType.TIME && !result.measureMemory) { + throw new IncompatibleArgumentsException( + "--primaryMeasurementType " + result.primaryMeasurementType.toString().toLowerCase()); + } + + return result; + } + + public static void printUsage() { + System.out.println(); + System.out.println("Usage: Runner [OPTIONS...] "); + System.out.println(); + System.out.println(" : a benchmark class or suite"); + System.out.println(); + System.out.println("OPTIONS"); + System.out.println(); + System.out.println(" -D=: fix a benchmark parameter to a given value."); + System.out.println(" Multiple values can be supplied by separating them with the"); + System.out.println(" delimiter specified in the --delimiter argument."); + System.out.println(); + System.out.println(" For example: \"-Dfoo=bar,baz,bat\""); + System.out.println(); + System.out.println(" \"benchmark\" is a special parameter that can be used to specify"); + System.out.println(" which benchmark methods to run. For example, if a benchmark has"); + System.out.println(" the method \"timeFoo\", it can be run alone by using"); + System.out.println(" \"-Dbenchmark=Foo\". \"benchmark\" also accepts a delimiter"); + System.out.println(" separated list of methods to run."); + System.out.println(); + System.out.println(" -J=: set a JVM argument to the given value."); + System.out.println(" Multiple values can be supplied by separating them with the"); + System.out.println(" delimiter specified in the --delimiter argument."); + System.out.println(); + System.out.println(" For example: \"-JmemoryMax=-Xmx32M,-Xmx512M\""); + System.out.println(); + System.out.println(" --delimiter : character or string to use as a delimiter"); + System.out.println(" for parameter and vm values."); + System.out.println(" Default: \"" + defaultDelimiter + "\""); + System.out.println(); + System.out.println(" --warmupMillis : duration to warmup each benchmark"); + System.out.println(); + System.out.println(" --runMillis : duration to execute each benchmark"); + System.out.println(); + System.out.println(" --captureVmLog: record the VM's just-in-time compiler and GC logs."); + System.out.println(" This may slow down or break benchmark display tools."); + System.out.println(); + System.out.println(" --measureMemory: measure the number of allocations done and the amount of"); + System.out.println(" memory used by invocations of the benchmark."); + System.out.println(" Default: off"); + System.out.println(); + System.out.println(" --vm : executable to test benchmark on. Multiple VMs may be passed"); + System.out.println(" in as a list separated by the delimiter specified in the"); + System.out.println(" --delimiter argument."); + System.out.println(); + System.out.println(" --timeUnit : unit of time to use for result. Depends on the units"); + System.out.println(" defined in the benchmark's getTimeUnitNames() method, if defined."); + System.out.println(" Default Options: ns, us, ms, s"); + System.out.println(); + System.out.println(" --instanceUnit : unit to use for allocation instances result."); + System.out.println(" Depends on the units defined in the benchmark's"); + System.out.println(" getInstanceUnitNames() method, if defined."); + System.out.println(" Default Options: instances, K instances, M instances, B instances"); + System.out.println(); + System.out.println(" --memoryUnit : unit to use for allocation memory size result."); + System.out.println(" Depends on the units defined in the benchmark's"); + System.out.println(" getMemoryUnitNames() method, if defined."); + System.out.println(" Default Options: B, KB, MB, GB"); + System.out.println(); + System.out.println(" --saveResults : write results to this file or directory"); + System.out.println(); + System.out.println(" --printScore: if present, also display an aggregate score for this run,"); + System.out.println(" where higher is better. This number has no particular meaning,"); + System.out.println(" but can be compared to scores from other runs that use the exact"); + System.out.println(" same arguments."); + System.out.println(); + System.out.println(" --uploadResults : upload this file or directory of files"); + System.out.println(" to the web app. This argument ends Caliper early and is thus"); + System.out.println(" incompatible with all other arguments."); + System.out.println(); + System.out.println(" --debug: run without measurement for use with debugger or profiling."); + System.out.println(); + System.out.println(" --debug-reps: fixed number of reps to run with --debug."); + System.out.println(" Default: \"" + defaultDebugReps + "\""); + + // adding new options? don't forget to update executeForked() + } +} diff --git a/caliper/src/main/java/com/google/caliper/Benchmark.java b/caliper/src/main/java/com/google/caliper/Benchmark.java new file mode 100644 index 0000000..06bda82 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Benchmark.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import java.util.Map; +import java.util.Set; + +/** + * A collection of benchmarks that share a set of configuration parameters. + */ +public interface Benchmark { + + Set parameterNames(); + + Set parameterValues(String parameterName); + + ConfiguredBenchmark createBenchmark(Map parameterValues); + + /** + * A mapping of units to their values. Their values must be integers, but all values are relative, + * so if one unit is 1.5 times the size of another, then these units can be expressed as + * {"unit1"=10,"unit2"=15}. The smallest unit given by the function will be used to display + * immediate results when running at the command line. + * + * e.g. 0% Scenario{...} 16.08; σ=1.72 @ 3 trials + */ + Map getTimeUnitNames(); + + Map getInstanceUnitNames(); + + Map getMemoryUnitNames(); + + /** + * Converts nanoseconds to the smallest unit defined in {@link #getTimeUnitNames()}. + */ + double nanosToUnits(double nanos); + + double instancesToUnits(long instances); + + double bytesToUnits(long bytes); +} \ No newline at end of file diff --git a/caliper/src/main/java/com/google/caliper/CaliperRc.java b/caliper/src/main/java/com/google/caliper/CaliperRc.java new file mode 100644 index 0000000..21a283f --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/CaliperRc.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class CaliperRc { + public static final CaliperRc INSTANCE = new CaliperRc(); + + private final Properties properties = new Properties(); + + private CaliperRc() { + try { + String caliperRcEnvVar = System.getenv("CALIPERRC"); + File caliperRcFile = (caliperRcEnvVar == null) + ? new File(System.getProperty("user.home"), ".caliperrc") + : new File(caliperRcEnvVar); + if (caliperRcFile.exists()) { + InputStream in = new FileInputStream(caliperRcFile); + properties.load(in); + in.close(); + } else { + // create it with a template + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String getApiKey() { + return properties.getProperty("apiKey"); + } + + public String getPostUrl() { + return properties.getProperty("postUrl"); + } + + /** + * The HTTP proxy host name and port number separated by a colon, such as + * foo.com:8080 + */ + public String getProxy() { + return properties.getProperty("proxy"); + } +} diff --git a/caliper/src/main/java/com/google/caliper/ConfigurationException.java b/caliper/src/main/java/com/google/caliper/ConfigurationException.java new file mode 100644 index 0000000..c4a35ec --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/ConfigurationException.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +/** + * Thrown upon occurrence of a configuration error. + */ +final class ConfigurationException extends RuntimeException { + + ConfigurationException(String s) { + super(s); + } + + ConfigurationException(Throwable cause) { + super(cause); + } + + private static final long serialVersionUID = 0; +} diff --git a/caliper/src/main/java/com/google/caliper/ConfiguredBenchmark.java b/caliper/src/main/java/com/google/caliper/ConfiguredBenchmark.java new file mode 100644 index 0000000..f150ec1 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/ConfiguredBenchmark.java @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import java.util.Map; + +public abstract class ConfiguredBenchmark { + + private final Benchmark underlyingBenchmark; + + protected ConfiguredBenchmark(Benchmark underlyingBenchmark) { + this.underlyingBenchmark = underlyingBenchmark; + } + + /** + * Runs the benchmark through {@code reps} iterations. + * + * @return any object or null. Benchmark implementors may keep an accumulating + * value to prevent the runtime from optimizing away the code under test. + * Such an accumulator value can be returned here. + */ + public abstract Object run(int reps) throws Exception; + + public abstract void close() throws Exception; + + public final Benchmark getBenchmark() { + return underlyingBenchmark; + } + + public final double nanosToUnits(double nanos) { + return underlyingBenchmark.nanosToUnits(nanos); + } + + public final Map timeUnitNames() { + return underlyingBenchmark.getTimeUnitNames(); + } + + public final double instancesToUnits(long instances) { + return underlyingBenchmark.instancesToUnits(instances); + } + + public final Map instanceUnitNames() { + return underlyingBenchmark.getInstanceUnitNames(); + } + + public final double bytesToUnits(long bytes) { + return underlyingBenchmark.bytesToUnits(bytes); + } + + public final Map memoryUnitNames() { + return underlyingBenchmark.getMemoryUnitNames(); + } +} diff --git a/caliper/src/main/java/com/google/caliper/ConsoleReport.java b/caliper/src/main/java/com/google/caliper/ConsoleReport.java new file mode 100644 index 0000000..8449bda --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/ConsoleReport.java @@ -0,0 +1,427 @@ +/** + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.caliper.util.LinearTranslation; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Ordering; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Prints a report containing the tested values and the corresponding + * measurements. Measurements are grouped by variable using indentation. + * Alongside numeric values, quick-glance ascii art bar charts are printed. + * Sample output (this may not represent the exact form that is produced): + * 
+ *              benchmark          d     ns linear runtime
+ * ConcatenationBenchmark 3.14159265   4397 ========================
+ * ConcatenationBenchmark       -0.0    223 ===============
+ *     FormatterBenchmark 3.14159265  33999 ==============================
+ *     FormatterBenchmark       -0.0  26399 =============================
+ *
+ */ +final class ConsoleReport { + + private static final int barGraphWidth = 30; + + private static final int UNITS_FOR_SCORE_100 = 1; + private static final int UNITS_FOR_SCORE_10 = 1000000000; // 1 s + + private static final LinearTranslation scoreTranslation = + new LinearTranslation(Math.log(UNITS_FOR_SCORE_10), 10, + Math.log(UNITS_FOR_SCORE_100), 100); + + public static final Ordering> UNIT_ORDERING = + new Ordering>() { + @Override public int compare(Entry a, Entry b) { + return a.getValue().compareTo(b.getValue()); + } + }; + + private final List variables; + private final Run run; + private final List scenarios; + + private final List orderedMeasurementTypes; + private final MeasurementType type; + private final double maxValue; + private final double logMinValue; + private final double logMaxValue; + private final EnumMap decimalDigitsMap = + new EnumMap(MeasurementType.class); + private final EnumMap divideByMap = + new EnumMap(MeasurementType.class); + private final EnumMap unitMap = + new EnumMap(MeasurementType.class); + private final EnumMap measurementColumnLengthMap = + new EnumMap(MeasurementType.class); + private boolean printScore; + + ConsoleReport(Run run, Arguments arguments) { + this.run = run; + unitMap.put(MeasurementType.TIME, arguments.getTimeUnit()); + unitMap.put(MeasurementType.INSTANCE, arguments.getInstanceUnit()); + unitMap.put(MeasurementType.MEMORY, arguments.getMemoryUnit()); + + if (arguments.getMeasureMemory()) { + orderedMeasurementTypes = Arrays.asList( + MeasurementType.TIME, MeasurementType.INSTANCE, MeasurementType.MEMORY); + } else { + orderedMeasurementTypes = Arrays.asList(MeasurementType.TIME); + } + + if (arguments.getPrimaryMeasurementType() != null) { + this.type = arguments.getPrimaryMeasurementType(); + } else { + this.type = MeasurementType.TIME; + } + + double min = Double.POSITIVE_INFINITY; + double max = 0; + + Multimap nameToValues = LinkedHashMultimap.create(); + List variablesBuilder = new ArrayList(); + for (Entry entry : this.run.getMeasurements().entrySet()) { + Scenario scenario = entry.getKey(); + double d = entry.getValue().getMeasurementSet(type).medianUnits(); + + min = Math.min(min, d); + max = Math.max(max, d); + + for (Entry variable : scenario.getVariables().entrySet()) { + String name = variable.getKey(); + nameToValues.put(name, variable.getValue()); + } + } + + for (Entry> entry : nameToValues.asMap().entrySet()) { + Variable variable = new Variable(entry.getKey(), entry.getValue()); + variablesBuilder.add(variable); + } + + /* + * Figure out how much influence each variable has on the measured value. + * We sum the measurements taken with each value of each variable. For + * variable that have influence on the measurement, the sums will differ + * by value. If the variable has little influence, the sums will be similar + * to one another and close to the overall average. We take the standard + * deviation across each variable's collection of sums. Higher standard + * deviation implies higher influence on the measured result. + */ + double sumOfAllMeasurements = 0; + for (ScenarioResult measurement : this.run.getMeasurements().values()) { + sumOfAllMeasurements += measurement.getMeasurementSet(type).medianUnits(); + } + for (Variable variable : variablesBuilder) { + int numValues = variable.values.size(); + double[] sumForValue = new double[numValues]; + for (Entry entry + : this.run.getMeasurements().entrySet()) { + Scenario scenario = entry.getKey(); + sumForValue[variable.index(scenario)] += + entry.getValue().getMeasurementSet(type).medianUnits(); + } + double mean = sumOfAllMeasurements / sumForValue.length; + double stdDeviationSquared = 0; + for (double value : sumForValue) { + double distance = value - mean; + stdDeviationSquared += distance * distance; + } + variable.stdDeviation = Math.sqrt(stdDeviationSquared / numValues); + } + + this.variables = new StandardDeviationOrdering().reverse().sortedCopy(variablesBuilder); + this.scenarios = new ByVariablesOrdering().sortedCopy(this.run.getMeasurements().keySet()); + this.maxValue = max; + this.logMinValue = Math.log(min); + this.logMaxValue = Math.log(max); + + EnumMap digitsBeforeDecimalMap = + new EnumMap(MeasurementType.class); + EnumMap decimalPointMap = + new EnumMap(MeasurementType.class); + for (MeasurementType measurementType : orderedMeasurementTypes) { + double maxForType = 0; + double minForType = Double.POSITIVE_INFINITY; + for (Entry entry : this.run.getMeasurements().entrySet()) { + double d = entry.getValue().getMeasurementSet(measurementType).medianUnits(); + minForType = Math.min(minForType, d); + maxForType = Math.max(maxForType, d); + } + + unitMap.put(measurementType, + getUnit(unitMap.get(measurementType), measurementType, minForType)); + + divideByMap.put(measurementType, + (double) getUnits(measurementType).get(unitMap.get(measurementType))); + + int numDigitsInMin = ceil(Math.log10(minForType)); + decimalDigitsMap.put(measurementType, + ceil(Math.max(0, ceil(Math.log10(divideByMap.get(measurementType))) + 3 - numDigitsInMin))); + + digitsBeforeDecimalMap.put(measurementType, + Math.max(1, ceil(Math.log10(maxForType / divideByMap.get(measurementType))))); + + decimalPointMap.put(measurementType, decimalDigitsMap.get(measurementType) > 0 ? 1 : 0); + + measurementColumnLengthMap.put(measurementType, Math.max(maxForType > 0 + ? digitsBeforeDecimalMap.get(measurementType) + decimalPointMap.get(measurementType) + + decimalDigitsMap.get(measurementType) + : 1, unitMap.get(measurementType).trim().length())); + } + + this.printScore = arguments.printScore(); + } + + private String getUnit(String userSuppliedUnit, MeasurementType measurementType, double min) { + Map units = getUnits(measurementType); + + if (userSuppliedUnit == null) { + List> entries = UNIT_ORDERING.reverse().sortedCopy(units.entrySet()); + for (Entry entry : entries) { + if (min / entry.getValue() >= 1) { + return entry.getKey(); + } + } + // if no unit works, just use the smallest available unit. + return entries.get(entries.size() - 1).getKey(); + } + + if (!units.keySet().contains(userSuppliedUnit)) { + throw new RuntimeException("\"" + unitMap.get(measurementType) + "\" is not a valid unit."); + } + return userSuppliedUnit; + } + + private Map getUnits(MeasurementType measurementType) { + Map units = null; + for (Entry entry : run.getMeasurements().entrySet()) { + if (units == null) { + units = entry.getValue().getMeasurementSet(measurementType).getUnitNames(); + } else { + if (!units.equals(entry.getValue().getMeasurementSet(measurementType).getUnitNames())) { + throw new RuntimeException("measurement sets for run contain multiple, incompatible unit" + + " sets."); + } + } + } + if (units == null) { + throw new RuntimeException("run has no measurements."); + } + if (units.isEmpty()) { + throw new RuntimeException("no units specified."); + } + return units; + } + + /** + * A variable and the set of values to which it has been assigned. + */ + private static class Variable { + final String name; + final ImmutableList values; + final int maxLength; + double stdDeviation; + + Variable(String name, Collection values) { + this.name = name; + this.values = ImmutableList.copyOf(values); + + int maxLen = name.length(); + for (String value : values) { + maxLen = Math.max(maxLen, value.length()); + } + this.maxLength = maxLen; + } + + String get(Scenario scenario) { + return scenario.getVariables().get(name); + } + + int index(Scenario scenario) { + return values.indexOf(get(scenario)); + } + + boolean isInteresting() { + return values.size() > 1; + } + } + + /** + * Orders the different variables by their standard deviation. This results + * in an appropriate grouping of output values. + */ + private static class StandardDeviationOrdering extends Ordering { + public int compare(Variable a, Variable b) { + return Double.compare(a.stdDeviation, b.stdDeviation); + } + } + + /** + * Orders scenarios by the variables. + */ + private class ByVariablesOrdering extends Ordering { + public int compare(Scenario a, Scenario b) { + for (Variable variable : variables) { + int aValue = variable.values.indexOf(variable.get(a)); + int bValue = variable.values.indexOf(variable.get(b)); + int diff = aValue - bValue; + if (diff != 0) { + return diff; + } + } + return 0; + } + } + + void displayResults() { + printValues(); + System.out.println(); + printUninterestingVariables(); + printCharCounts(); + } + + private void printCharCounts() { + int systemOutCharCount = 0; + int systemErrCharCount = 0; + for (ScenarioResult scenarioResult : run.getMeasurements().values()) { + for (MeasurementType measurementType : MeasurementType.values()) { + MeasurementSet measurementSet = scenarioResult.getMeasurementSet(measurementType); + if (measurementSet != null) { + systemOutCharCount += measurementSet.getSystemOutCharCount(); + systemErrCharCount += measurementSet.getSystemErrCharCount(); + } + } + } + if (systemOutCharCount > 0 || systemErrCharCount > 0) { + System.out.println(); + System.out.println("Note: benchmarks printed " + systemOutCharCount + + " characters to System.out and " + systemErrCharCount + " characters to System.err." + + " Use --debug to see this output."); + } + } + + /** + * Prints a table of values. + */ + private void printValues() { + // header + for (Variable variable : variables) { + if (variable.isInteresting()) { + System.out.printf("%" + variable.maxLength + "s ", variable.name); + } + } + // doesn't make sense to show graphs at all for 1 + // scenario, since it leads to vacuous graphs. + boolean showGraphs = scenarios.size() > 1; + + EnumMap numbersFormatMap = + new EnumMap(MeasurementType.class); + for (MeasurementType measurementType : orderedMeasurementTypes) { + if (measurementType != type) { + System.out.printf("%" + measurementColumnLengthMap.get(measurementType) + "s ", + unitMap.get(measurementType).trim()); + } + + numbersFormatMap.put(measurementType, + "%" + measurementColumnLengthMap.get(measurementType) + + "." + decimalDigitsMap.get(measurementType) + "f" + + (type == measurementType ? "" : " ")); + } + + System.out.printf("%" + measurementColumnLengthMap.get(type) + "s", unitMap.get(type).trim()); + if (showGraphs) { + System.out.print(" linear runtime"); + } + System.out.println(); + + double sumOfLogs = 0.0; + + for (Scenario scenario : scenarios) { + for (Variable variable : variables) { + if (variable.isInteresting()) { + System.out.printf("%" + variable.maxLength + "s ", variable.get(scenario)); + } + } + ScenarioResult measurement = run.getMeasurements().get(scenario); + sumOfLogs += Math.log(measurement.getMeasurementSet(type).medianUnits()); + + for (MeasurementType measurementType : orderedMeasurementTypes) { + if (measurementType != type) { + System.out.printf(numbersFormatMap.get(measurementType), + measurement.getMeasurementSet(measurementType).medianUnits() / divideByMap.get(measurementType)); + } + } + + System.out.printf(numbersFormatMap.get(type), + measurement.getMeasurementSet(type).medianUnits() / divideByMap.get(type)); + if (showGraphs) { + System.out.printf(" %s", barGraph(measurement.getMeasurementSet(type).medianUnits())); + } + System.out.println(); + } + + if (printScore) { + // arithmetic mean of logs, aka log of geometric mean + double meanLogUnits = sumOfLogs / scenarios.size(); + System.out.format("%nScore: %.3f%n", scoreTranslation.translate(meanLogUnits)); + } + } + + /** + * Prints variables with only one unique value. + */ + private void printUninterestingVariables() { + for (Variable variable : variables) { + if (!variable.isInteresting()) { + System.out.println(variable.name + ": " + Iterables.getOnlyElement(variable.values)); + } + } + } + + /** + * Returns a string containing a bar of proportional width to the specified + * value. + */ + private String barGraph(double value) { + int graphLength = floor(value / maxValue * barGraphWidth); + graphLength = Math.max(1, graphLength); + graphLength = Math.min(barGraphWidth, graphLength); + return Strings.repeat("=", graphLength); + } + + @SuppressWarnings("NumericCastThatLosesPrecision") + private static int floor(double d) { + return (int) d; + } + + @SuppressWarnings("NumericCastThatLosesPrecision") + private static int ceil(double d) { + return (int) Math.ceil(d); + } +} diff --git a/caliper/src/main/java/com/google/caliper/CountingPrintStream.java b/caliper/src/main/java/com/google/caliper/CountingPrintStream.java new file mode 100644 index 0000000..5c19a61 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/CountingPrintStream.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import java.io.PrintStream; + +/** + * Counts how many characters were written. + */ +final class CountingPrintStream extends PrintStream { + + private final PrintStream delegate; + private int count; + + CountingPrintStream(PrintStream delegate) { + super(delegate); + this.delegate = delegate; + } + + public int getCount() { + return count; + } + + @Override public void flush() { + delegate.flush(); + } + + @Override public void close() { + delegate.close(); + } + + @Override public boolean checkError() { + return delegate.checkError(); + } + + @Override protected void setError() { + throw new UnsupportedOperationException(); + } + + @Override protected void clearError() { + throw new UnsupportedOperationException(); + } + + @Override public void write(int b) { + count++; + delegate.write(b); + } + + @Override public void write(byte[] buffer, int offset, int length) { + count += length; + delegate.write(buffer, offset, length); + } + + @Override public void print(char[] chars) { + count += chars.length; + delegate.print(chars); + } + + @Override public void print(String s) { + count += s.length(); + delegate.print(s); + } +} diff --git a/caliper/src/main/java/com/google/caliper/DalvikVm.java b/caliper/src/main/java/com/google/caliper/DalvikVm.java new file mode 100644 index 0000000..f89cc28 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/DalvikVm.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableList; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +/** + * The dalvikvm run on Android devices via the app_process executable. + */ +final class DalvikVm extends Vm { + + public static boolean isDalvikVm() { + return "Dalvik".equals(System.getProperty("java.vm.name")); + } + + public static String vmName() { + return "app_process"; + } + + @Override public List getVmSpecificOptions(MeasurementType type, Arguments arguments) { + if (!arguments.getCaptureVmLog()) { + return ImmutableList.of(); + } + + List result = new ArrayList(); + if (arguments.getCaptureVmLog()) { + // TODO: currently GC goes to logcat. + // result.add("-verbose:gc"); + } + return result; + } + + @Override public ProcessBuilder newProcessBuilder(File workingDirectory, String classPath, + ImmutableList vmArgs, String className, ImmutableList applicationArgs) { + ProcessBuilder result = new ProcessBuilder(); + result.directory(workingDirectory); + result.command().addAll(vmArgs); + result.command().add("-Djava.class.path=" + classPath); + result.command().add(workingDirectory.getPath()); + result.command().add(className); + result.command().addAll(applicationArgs); + return result; + } +} diff --git a/caliper/src/main/java/com/google/caliper/DebugMeasurer.java b/caliper/src/main/java/com/google/caliper/DebugMeasurer.java new file mode 100644 index 0000000..64d4b7f --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/DebugMeasurer.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.base.Supplier; + +/** + * Measure's the benchmark's per-trial execution time. + */ +class DebugMeasurer extends Measurer { + + private final int reps; + + DebugMeasurer(int reps) { + checkArgument(reps > 0); + this.reps = reps; + } + + @Override public MeasurementSet run(Supplier testSupplier) + throws Exception { + ConfiguredBenchmark benchmark = testSupplier.get(); + benchmark.run(reps); + benchmark.close(); + return null; + } +} diff --git a/caliper/src/main/java/com/google/caliper/Environment.java b/caliper/src/main/java/com/google/caliper/Environment.java new file mode 100644 index 0000000..75c2813 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Environment.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.annotations.GwtCompatible; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +/** + * A description of an environment in which benchmarks are run. + * + * WARNING: a JSON representation of this class is stored on the app engine server. If any changes + * are made to this class, a deserialization adapter must be written for this class to ensure + * backwards compatibility. + * + *

Gwt-safe + */ +@SuppressWarnings("serial") +@GwtCompatible +public final class Environment + implements Serializable /* for GWT Serialization */ { + private /*final*/ Map propertyMap; + + public Environment(Map propertyMap) { + this.propertyMap = new HashMap(propertyMap); + } + + public Map getProperties() { + return propertyMap; + } + + @Override public boolean equals(Object o) { + return o instanceof Environment + && ((Environment) o).propertyMap.equals(propertyMap); + } + + @Override public int hashCode() { + return propertyMap.hashCode(); + } + + @Override public String toString() { + return propertyMap.toString(); + } + + @SuppressWarnings("unused") + private Environment() {} // for GWT Serialization +} diff --git a/caliper/src/main/java/com/google/caliper/EnvironmentGetter.java b/caliper/src/main/java/com/google/caliper/EnvironmentGetter.java new file mode 100644 index 0000000..9b7150b --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/EnvironmentGetter.java @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableMultiset; +import com.google.common.collect.Multimap; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class EnvironmentGetter { + + public Environment getEnvironmentSnapshot() { + Map propertyMap = new HashMap(); + + @SuppressWarnings("unchecked") + Map sysProps = (Map) (Map) System.getProperties(); + + // Sometimes java.runtime.version is more descriptive than java.version + String version = sysProps.get("java.version"); + String alternateVersion = sysProps.get("java.runtime.version"); + if (alternateVersion != null && alternateVersion.length() > version.length()) { + version = alternateVersion; + } + propertyMap.put("jre.version", version); + + propertyMap.put("jre.vmname", sysProps.get("java.vm.name")); + propertyMap.put("jre.vmversion", sysProps.get("java.vm.version")); + propertyMap.put("jre.availableProcessors", + Integer.toString(Runtime.getRuntime().availableProcessors())); + + String osName = sysProps.get("os.name"); + propertyMap.put("os.name", osName); + propertyMap.put("os.version", sysProps.get("os.version")); + propertyMap.put("os.arch", sysProps.get("os.arch")); + + try { + propertyMap.put("host.name", InetAddress.getLocalHost().getHostName()); + } catch (UnknownHostException ignored) { + } + + if (osName.equals("Linux")) { + getLinuxEnvironment(propertyMap); + } + + return new Environment(propertyMap); + } + + private void getLinuxEnvironment(Map propertyMap) { + // the following probably doesn't work on ALL linux + Multimap cpuInfo = propertiesFromLinuxFile("/proc/cpuinfo"); + propertyMap.put("host.cpus", Integer.toString(cpuInfo.get("processor").size())); + String s = "cpu cores"; + propertyMap.put("host.cpu.cores", describe(cpuInfo, s)); + propertyMap.put("host.cpu.names", describe(cpuInfo, "model name")); + propertyMap.put("host.cpu.cachesize", describe(cpuInfo, "cache size")); + + Multimap memInfo = propertiesFromLinuxFile("/proc/meminfo"); + // TODO redo memInfo.toString() so we don't get square brackets + propertyMap.put("host.memory.physical", memInfo.get("MemTotal").toString()); + propertyMap.put("host.memory.swap", memInfo.get("SwapTotal").toString()); + + getAndroidEnvironment(propertyMap); + } + + private void getAndroidEnvironment(Map propertyMap) { + try { + Map map = getAndroidProperties(); + String manufacturer = map.get("ro.product.manufacturer"); + String device = map.get("ro.product.device"); + propertyMap.put("android.device", manufacturer + " " + device); // "Motorola sholes" + + String brand = map.get("ro.product.brand"); + String model = map.get("ro.product.model"); + propertyMap.put("android.model", brand + " " + model); // "verizon Droid" + + String release = map.get("ro.build.version.release"); + String id = map.get("ro.build.id"); + propertyMap.put("android.release", release + " " + id); // "Gingerbread GRH07B" + } catch (IOException ignored) { + } + } + + private static String describe(Multimap cpuInfo, String s) { + Collection strings = cpuInfo.get(s); + // TODO redo the ImmutableMultiset.toString() call so we don't get square brackets + return (strings.size() == 1) + ? strings.iterator().next() + : ImmutableMultiset.copyOf(strings).toString(); + } + + /** + * Returns the key/value pairs from the specified properties-file like + * reader. Unlike standard Java properties files, {@code reader} is allowed + * to list the same property multiple times. Comments etc. are unsupported. + */ + private static Multimap propertiesFileToMultimap(Reader reader) + throws IOException { + ImmutableMultimap.Builder result = ImmutableMultimap.builder(); + BufferedReader in = new BufferedReader(reader); + + String line; + while((line = in.readLine()) != null) { + String[] parts = line.split("\\s*\\:\\s*", 2); + if (parts.length == 2) { + result.put(parts[0], parts[1]); + } + } + in.close(); + + return result.build(); + } + + private static Multimap propertiesFromLinuxFile(String file) { + try { + Process process = Runtime.getRuntime().exec(new String[]{"/bin/cat", file}); + return propertiesFileToMultimap( + new InputStreamReader(process.getInputStream(), "ISO-8859-1")); + } catch (IOException e) { + return ImmutableMultimap.of(); + } + } + + public static void main(String[] args) { + Environment snapshot = new EnvironmentGetter().getEnvironmentSnapshot(); + for (Map.Entry entry : snapshot.getProperties().entrySet()) { + System.out.println(entry.getKey() + " " + entry.getValue()); + } + } + + /** + * Android properties are available from adb shell /system/bin/getprop. That + * program prints Android system properties in this format: + * [ro.product.model]: [Droid] + * [ro.product.brand]: [verizon] + */ + private static Map getAndroidProperties() throws IOException { + Map result = new HashMap(); + + Process process = Runtime.getRuntime().exec(new String[] {"/system/bin/getprop"}); + BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), "ISO-8859-1")); + + Pattern pattern = Pattern.compile("\\[([^\\]]*)\\]: \\[([^\\]]*)\\]"); + String line; + while ((line = reader.readLine()) != null) { + Matcher matcher = pattern.matcher(line); + if (matcher.matches()) { + result.put(matcher.group(1), matcher.group(2)); + } + } + return result; + } +} diff --git a/caliper/src/main/java/com/google/caliper/InProcessRunner.java b/caliper/src/main/java/com/google/caliper/InProcessRunner.java new file mode 100644 index 0000000..1ef4788 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/InProcessRunner.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.caliper.UserException.ExceptionFromUserCodeException; +import com.google.common.base.Supplier; + +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.List; + +/** + * Executes a benchmark in the current VM. + */ +final class InProcessRunner { + + public void run(String... args) { + Arguments arguments = Arguments.parse(args); + + ScenarioSelection scenarioSelection = new ScenarioSelection(arguments); + + try { + Measurer measurer = getMeasurer(arguments); + List scenarios = scenarioSelection.select(); + // We only expect one scenario right now - if we have more, something has gone wrong. + // This matters for things like reading the measurements. This is only done once, so if + // multiple scenarios are executed, they will be ignored! + if (scenarios.size() != 1) { + throw new IllegalArgumentException("Invalid arguments to subprocess. Expected exactly one " + + "scenario but got " + scenarios.size()); + } + Scenario scenario = scenarios.get(0); + + System.out.println("starting " + scenario); + MeasurementSet measurementSet = run(scenarioSelection, scenario, measurer); + System.out.println(arguments.getMarker() + Json.measurementSetToJson(measurementSet)); + } catch (UserException e) { + throw e; + } catch (Exception e) { + throw new ExceptionFromUserCodeException(e); + } + } + + public MeasurementSet run(final ScenarioSelection scenarioSelection, final Scenario scenario, + Measurer measurer) throws Exception { + Supplier supplier = new Supplier() { + @Override public ConfiguredBenchmark get() { + return scenarioSelection.createBenchmark(scenario); + } + }; + + PrintStream out = System.out; + PrintStream err = System.err; + measurer.setLogStream(out); + CountingPrintStream countedOut = new CountingPrintStream(out); + CountingPrintStream countedErr = new CountingPrintStream(err); + System.setOut(countedOut); + System.setErr(countedErr); + try { + MeasurementSet measurementSet = measurer.run(supplier); + if (measurementSet != null) { + measurementSet = measurementSet.plusCharCounts( + countedOut.getCount(), countedErr.getCount()); + } + return measurementSet; + } finally { + System.setOut(out); + System.setErr(err); + } + } + + private Measurer getMeasurer(Arguments arguments) { + if (arguments.getMeasurementType() == MeasurementType.TIME) { + return new TimeMeasurer(arguments.getWarmupMillis(), arguments.getRunMillis()); + } else if (arguments.getMeasurementType() == MeasurementType.INSTANCE) { + return new InstancesAllocationMeasurer(); + } else if (arguments.getMeasurementType() == MeasurementType.MEMORY) { + return new MemoryAllocationMeasurer(); + } else if (arguments.getMeasurementType() == MeasurementType.DEBUG) { + return new DebugMeasurer(arguments.getDebugReps()); + } else { + throw new IllegalArgumentException("unrecognized measurement type: " + + arguments.getMeasurementType()); + } + } + + public static void main(String... args) throws Exception { + try { + new InProcessRunner().run(args); + System.exit(0); // user code may have leave non-daemon threads behind! + } catch (UserException e) { + e.display(); // TODO: send this to the host process + System.out.println(LogConstants.CALIPER_LOG_PREFIX + LogConstants.SCENARIOS_FINISHED); + System.exit(1); + } + } + + public PrintStream nullPrintStream() { + return new PrintStream(new OutputStream() { + public void write(int b) {} + }); + } +} diff --git a/caliper/src/main/java/com/google/caliper/InstancesAllocationMeasurer.java b/caliper/src/main/java/com/google/caliper/InstancesAllocationMeasurer.java new file mode 100644 index 0000000..3fe89c7 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/InstancesAllocationMeasurer.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +public final class InstancesAllocationMeasurer extends AllocationMeasurer { + + InstancesAllocationMeasurer() { + type = "instance"; + } + + @Override protected long incrementAllocationCount(long oldAllocationCount, int arrayCount, + long size) { + return oldAllocationCount + 1; + } + + @Override protected Measurement getMeasurement(ConfiguredBenchmark benchmark, long allocations) { + return new Measurement(benchmark.instanceUnitNames(), allocations, + benchmark.instancesToUnits(allocations)); + } +} diff --git a/caliper/src/main/java/com/google/caliper/Json.java b/caliper/src/main/java/com/google/caliper/Json.java new file mode 100644 index 0000000..9e3f809 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Json.java @@ -0,0 +1,226 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import com.google.gson.reflect.TypeToken; + +import java.lang.reflect.Type; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TimeZone; + +/** + * Ordinarily serialization should be done within the class that is being serialized. However, + * many of these classes are used by GWT, which dies when it sees Gson. + */ +public final class Json { + /** + * This Gson instance must be used when serializing a class that includes a Run as a member + * or as a member of a member (etc.), otherwise the Map will not + * be correctly serialized. + */ + private static final Gson GSON_INSTANCE = + new GsonBuilder() + .registerTypeAdapter(Date.class, new DateTypeAdapter()) + .registerTypeAdapter(Run.class, new RunTypeAdapter()) + .registerTypeAdapter(Measurement.class, new MeasurementDeserializer()) + .create(); + + public static Gson getGsonInstance() { + return GSON_INSTANCE; + } + + public static String measurementSetToJson(MeasurementSet measurementSet) { + return new Gson().toJson(measurementSet); + } + + /** + * Attempts to extract a MeasurementSet from a string, assuming it is JSON. If this fails, it + * tries to extract it from the string assuming it is a space-separated list of double values. + */ + public static MeasurementSet measurementSetFromJson(String measurementSetJson) { + try { + return getGsonInstance().fromJson(measurementSetJson, MeasurementSet.class); + } catch (JsonParseException e) { + // might be an old MeasurementSet, so fall back on failure to the old, space separated + // serialization method. + try { + String[] measurementStrings = measurementSetJson.split("\\s+"); + List measurements = new ArrayList(); + for (String s : measurementStrings) { + measurements.add( + new Measurement(ImmutableMap.of("ns", 1, "us", 1000, "ms", 1000000, "s", 1000000000), + Double.valueOf(s), Double.valueOf(s))); + } + // seconds and variations is the default unit + return new MeasurementSet(measurements.toArray(new Measurement[measurements.size()])); + } catch (NumberFormatException ignore) { + throw new IllegalArgumentException("Not a measurement set: " + measurementSetJson); + } + } + } + + public static MeasurementSet measurementSetFromJson(JsonObject measurementSetJson) { + return getGsonInstance().fromJson(measurementSetJson, MeasurementSet.class); + } + + /** + * Backwards compatibility! + */ + private static class MeasurementDeserializer implements JsonDeserializer { + @Override public Measurement deserialize(JsonElement jsonElement, Type type, + JsonDeserializationContext context) throws JsonParseException { + JsonObject obj = jsonElement.getAsJsonObject(); + if (obj.has("raw") && obj.has("processed")) { + return new Measurement( + context.>deserialize(obj.get("unitNames"), + new TypeToken>() {}.getType()), + context.deserialize(obj.get("raw"), Double.class), + context.deserialize(obj.get("processed"), Double.class)); + } + if (obj.has("nanosPerRep") && obj.has("unitsPerRep") && obj.has("unitNames")) { + return new Measurement( + context.>deserialize(obj.get("unitNames"), + new TypeToken>() {}.getType()), + context.deserialize(obj.get("nanosPerRep"), Double.class), + context.deserialize(obj.get("unitsPerRep"), Double.class)); + } + throw new JsonParseException(obj.toString()); + } + } + + /** + * This adapter is necessary because gson doesn't handle Maps more complex than Map + * in a useful way. For example, Map's serialized version simply uses + * Scenario.toString() as the keys. This adapter stores this Map as lists of + * KeyValuePair instead, to preserve the Scenario objects on + * deserialization. + */ + private static class RunTypeAdapter implements JsonSerializer, JsonDeserializer { + + @Override public Run deserialize(JsonElement jsonElement, Type type, + JsonDeserializationContext context) throws JsonParseException { + + List> mapList = context.deserialize( + jsonElement.getAsJsonObject().get("measurements"), + new TypeToken>>() {}.getType()); + Map measurements = new LinkedHashMap(); + for (KeyValuePair entry : mapList) { + measurements.put(entry.getKey(), entry.getValue()); + } + + String benchmarkName = + context.deserialize(jsonElement.getAsJsonObject().get("benchmarkName"), String.class); + + Date executedTimestamp = context.deserialize( + jsonElement.getAsJsonObject().get("executedTimestamp"), Date.class); + + return new Run(measurements, benchmarkName, executedTimestamp); + } + + @Override public JsonElement serialize(Run run, Type type, JsonSerializationContext context) { + JsonObject result = new JsonObject(); + result.add("benchmarkName", context.serialize(run.getBenchmarkName())); + result.add("executedTimestamp", context.serialize(run.getExecutedTimestamp())); + + List> mapList = + new ArrayList>(); + for (Map.Entry entry : run.getMeasurements().entrySet()) { + mapList.add(new KeyValuePair(entry.getKey(), entry.getValue())); + } + result.add("measurements", context.serialize(mapList, + new TypeToken>>() {}.getType())); + + return result; + } + } + + private static class DateTypeAdapter implements JsonSerializer, JsonDeserializer { + private final DateFormat dateFormat; + + private DateTypeAdapter() { + dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssz", Locale.US); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + @Override public synchronized JsonElement serialize(Date date, Type type, + JsonSerializationContext jsonSerializationContext) { + return new JsonPrimitive(dateFormat.format(date)); + } + + @Override public synchronized Date deserialize(JsonElement jsonElement, Type type, + JsonDeserializationContext jsonDeserializationContext) { + String dateString = jsonElement.getAsString(); + // first try to parse as an ISO 8601 date + try { + return dateFormat.parse(dateString); + } catch (ParseException ignored) { + } + // next, try a GSON-style locale-specific dates (for Caliper r282 and earlier) + try { + return DateFormat.getDateTimeInstance().parse(dateString); + } catch (ParseException ignored) { + } + throw new JsonParseException(dateString); + } + } + + /** + * This is similar to the Map.Entry class, but is necessary since Entrys are not supported + * by gson. + */ + private static class KeyValuePair { + private K k; + private V v; + + KeyValuePair(K k, V v) { + this.k = k; + this.v = v; + } + + public K getKey() { + return k; + } + + public V getValue() { + return v; + } + + @SuppressWarnings("unused") + private KeyValuePair() {} // for gson + } + + private Json() {} // static class +} diff --git a/caliper/src/main/java/com/google/caliper/LogConstants.java b/caliper/src/main/java/com/google/caliper/LogConstants.java new file mode 100644 index 0000000..09acebe --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/LogConstants.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +public final class LogConstants { + /** + * Must be prepended to line of XML that represents normalized scenario. + */ + public static final String SCENARIO_JSON_PREFIX = "[scenario] "; + public static final String MEASUREMENT_JSON_PREFIX = "[measurement] "; + + /** + * Must be prepended to any logs that are to be included in the run event log. + */ + public static final String CALIPER_LOG_PREFIX = "[caliper] "; + public static final String SCENARIOS_STARTING = "[starting scenarios]"; + public static final String STARTING_SCENARIO_PREFIX = "[starting scenario] "; + public static final String SCENARIO_FINISHED = "[scenario finished]"; + public static final String SCENARIOS_FINISHED = "[scenarios finished]"; + + /** + * All events will be logged from when {@code MEASURED_SECTION_STARTING} is logged until + * {@code MEASURED_SECTION_DONE} is logged. + */ + public static final String MEASURED_SECTION_STARTING = "[starting measured section]"; + public static final String MEASURED_SECTION_DONE = "[done measured section]"; + + private LogConstants() {} +} diff --git a/caliper/src/main/java/com/google/caliper/Measurement.java b/caliper/src/main/java/com/google/caliper/Measurement.java new file mode 100644 index 0000000..35865b6 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Measurement.java @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.annotations.GwtCompatible; + +import java.io.Serializable; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; + +/** + * Represents a measurement of a single run of a benchmark. + */ +@SuppressWarnings("serial") +@GwtCompatible +public final class Measurement + implements Serializable /* for GWT */ { + + public static final Comparator SORT_BY_NANOS = new Comparator() { + @Override public int compare(Measurement a, Measurement b) { + double aNanos = a.getRaw(); + double bNanos = b.getRaw(); + return Double.compare(aNanos, bNanos); + } + }; + + public static final Comparator SORT_BY_UNITS = new Comparator() { + @Override public int compare(Measurement a, Measurement b) { + double aNanos = a.getProcessed(); + double bNanos = b.getProcessed(); + return Double.compare(aNanos, bNanos); + } + }; + + private /*final*/ double raw; + private /*final*/ double processed; + private /*final*/ Map unitNames; + + public Measurement(Map unitNames, double raw, double processed) { + this.unitNames = new HashMap(unitNames); + this.raw = raw; + this.processed = processed; + } + + public Map getUnitNames() { + return new HashMap(unitNames); + } + + public double getRaw() { + return raw; + } + + public double getProcessed() { + return processed; + } + + @Override public boolean equals(Object o) { + return o instanceof Measurement + && ((Measurement) o).raw == raw + && ((Measurement) o).processed == processed + && ((Measurement) o).unitNames.equals(unitNames); + } + + @Override public int hashCode() { + return (int) raw // Double.doubleToLongBits doesn't exist on GWT + + (int) processed * 37 + + unitNames.hashCode() * 1373; + } + + @Override public String toString() { + return (raw != processed + ? raw + "/" + processed + : Double.toString(raw)); + } + + @SuppressWarnings("unused") + private Measurement() {} /* for GWT */ +} diff --git a/caliper/src/main/java/com/google/caliper/MeasurementSet.java b/caliper/src/main/java/com/google/caliper/MeasurementSet.java new file mode 100644 index 0000000..eb0b42a --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/MeasurementSet.java @@ -0,0 +1,264 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.annotations.GwtCompatible; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A collection of measurements of the same scenario. + */ +@SuppressWarnings("serial") +@GwtCompatible +public final class MeasurementSet + implements Serializable /* for GWT Serialization */ { + + private /*final*/ List measurements; + /** + * Mapping of user-defined units to relative sizes. + */ + private /*final*/ Map unitNames; + + private /*final*/ int systemOutCharCount; + private /*final*/ int systemErrCharCount; + + public MeasurementSet(Measurement... measurements) { + this(0, 0, getUnitNamesFromMeasurements(measurements), Arrays.asList(measurements)); + } + + private static Map getUnitNamesFromMeasurements(Measurement... measurements) { + Map unitNameToAssign = null; + for (Measurement measurement : measurements) { + if (unitNameToAssign == null) { + unitNameToAssign = new HashMap(measurement.getUnitNames()); + } else if (!unitNameToAssign.equals(measurement.getUnitNames())) { + throw new IllegalArgumentException("incompatible unit names: " + unitNameToAssign + " and " + + measurement.getUnitNames()); + } + } + return unitNameToAssign; + } + + /** + * Constructor to use directly from plusMeasurement. Skips some excessive checking and takes a + * list directly. + */ + private MeasurementSet(int systemOutCharCount, int systemErrCharCount, + Map unitNames, List measurements) { + this.systemOutCharCount = systemOutCharCount; + this.systemErrCharCount = systemErrCharCount; + this.unitNames = unitNames; + this.measurements = measurements; + } + + /** + * This is the same as getUnitNames(), but is for backwards compatibility on the server + * when null pointer exceptions need to be avoided. + */ + public Map getUnitNames(Map defaultValue) { + if (unitNames == null) { + return defaultValue; + } + return new HashMap(unitNames); + } + + public Map getUnitNames() { + return new HashMap(unitNames); + } + + public List getMeasurements() { + return new ArrayList(measurements); + } + + public int size() { + return measurements.size(); + } + + public int getSystemOutCharCount() { + return systemOutCharCount; + } + + public int getSystemErrCharCount() { + return systemErrCharCount; + } + + public List getMeasurementsRaw() { + List measurementRaw = new ArrayList(); + for (Measurement measurement : measurements) { + measurementRaw.add(measurement.getRaw()); + } + return measurementRaw; + } + + public List getMeasurementUnits() { + List measurementUnits = new ArrayList(); + for (Measurement measurement : measurements) { + measurementUnits.add(measurement.getProcessed()); + } + return measurementUnits; + } + + /** + * Returns the median measurement, with respect to raw units. + */ + public double medianRaw() { + return median(getMeasurementsRaw()); + } + + /** + * Returns the median measurement, with respect to user-defined units. + */ + public double medianUnits() { + return median(getMeasurementUnits()); + } + + private double median(List doubles) { + Collections.sort(doubles); + int n = doubles.size(); + return (n % 2 == 0) + ? (doubles.get(n / 2 - 1) + doubles.get(n / 2)) / 2 + : doubles.get(n / 2); + } + + /** + * Returns the average measurement with respect to raw units. + */ + public double meanRaw() { + return mean(getMeasurementsRaw()); + } + + /** + * Returns the average measurement with respect to user-defined units. + */ + public double meanUnits() { + return mean(getMeasurementUnits()); + } + + private double mean(List doubles) { + double sum = 0; + for (double d : doubles) { + sum += d; + } + return sum / doubles.size(); + } + + public double standardDeviationRaw() { + return standardDeviation(getMeasurementsRaw()); + } + + public double standardDeviationUnits() { + return standardDeviation(getMeasurementUnits()); + } + + /** + * Returns the standard deviation of the measurements. + */ + private double standardDeviation(List doubles) { + double mean = mean(doubles); + double sumOfSquares = 0; + for (double d : doubles) { + double delta = (d - mean); + sumOfSquares += (delta * delta); + } + return Math.sqrt(sumOfSquares / (doubles.size() - 1)); + } + + public double minRaw() { + return min(getMeasurementsRaw()); + } + + public double minUnits() { + return min(getMeasurementUnits()); + } + + /** + * Returns the minimum measurement. + */ + private double min(List doubles) { + Collections.sort(doubles); + return doubles.get(0); + } + + public double maxRaw() { + return max(getMeasurementsRaw()); + } + + public double maxUnits() { + return max(getMeasurementUnits()); + } + + /** + * Returns the maximum measurement. + */ + private double max(List doubles) { + Collections.sort(doubles, Collections.reverseOrder()); + return doubles.get(0); + } + + /** + * Returns a new measurement set that contains the measurements in this set + * plus the given additional measurement. + */ + public MeasurementSet plusMeasurement(Measurement measurement) { + // verify that this Measurement is compatible with this MeasurementSet + if (unitNames != null && !unitNames.equals(measurement.getUnitNames())) { + throw new IllegalArgumentException("new measurement incompatible with units of measurement " + + "set. Expected " + unitNames + " but got " + measurement.getUnitNames()); + } + + List resultMeasurements = new ArrayList(measurements); + resultMeasurements.add(measurement); + Map newUnitNames = unitNames == null ? measurement.getUnitNames() : unitNames; + return new MeasurementSet(systemOutCharCount, systemErrCharCount, + newUnitNames, resultMeasurements); + } + + public MeasurementSet plusCharCounts(int systemOutCharCount, int systemErrCharCount) { + return new MeasurementSet(this.systemOutCharCount + systemOutCharCount, + this.systemErrCharCount + systemErrCharCount, unitNames, measurements); + } + + @Override public boolean equals(Object o) { + return o instanceof MeasurementSet + && ((MeasurementSet) o).measurements.equals(measurements) + && ((MeasurementSet) o).unitNames.equals(unitNames) + && ((MeasurementSet) o).systemOutCharCount == systemOutCharCount + && ((MeasurementSet) o).systemErrCharCount == systemErrCharCount; + } + + @Override public int hashCode() { + return measurements.hashCode() + + unitNames.hashCode() * 37 + + systemOutCharCount * 1373 + + systemErrCharCount * 53549; + } + + @Override public String toString() { + return measurements.toString() + " " + unitNames + " " + + systemOutCharCount + "/" + systemErrCharCount; + } + + @SuppressWarnings("unused") + private MeasurementSet() {} // for GWT Serialization +} diff --git a/caliper/src/main/java/com/google/caliper/MeasurementType.java b/caliper/src/main/java/com/google/caliper/MeasurementType.java new file mode 100644 index 0000000..30de69f --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/MeasurementType.java @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.annotations.GwtCompatible; + +@GwtCompatible +public enum MeasurementType { + TIME, INSTANCE, MEMORY, DEBUG +} diff --git a/caliper/src/main/java/com/google/caliper/Measurer.java b/caliper/src/main/java/com/google/caliper/Measurer.java new file mode 100644 index 0000000..93002e6 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Measurer.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.base.Supplier; + +import java.io.PrintStream; + +abstract class Measurer { + + private PrintStream logStream = System.out; + + /** + * Sets the stream used to log caliper events. + */ + void setLogStream(PrintStream logStream) { + this.logStream = logStream; + } + + public abstract MeasurementSet run(Supplier testSupplier) throws Exception; + + protected void prepareForTest() { + System.gc(); + System.gc(); + } + + protected final void log(String message) { + logStream.println(LogConstants.CALIPER_LOG_PREFIX + message); + } +} diff --git a/caliper/src/main/java/com/google/caliper/MemoryAllocationMeasurer.java b/caliper/src/main/java/com/google/caliper/MemoryAllocationMeasurer.java new file mode 100644 index 0000000..a28aabf --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/MemoryAllocationMeasurer.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +public final class MemoryAllocationMeasurer extends AllocationMeasurer { + + public MemoryAllocationMeasurer() { + type = "byte"; + } + + @Override protected long incrementAllocationCount(long oldAllocationCount, int arrayCount, + long size) { + return oldAllocationCount + size; + } + + @Override protected Measurement getMeasurement(ConfiguredBenchmark benchmark, long allocations) { + return new Measurement(benchmark.memoryUnitNames(), allocations, + benchmark.bytesToUnits(allocations)); + } +} diff --git a/caliper/src/main/java/com/google/caliper/Param.java b/caliper/src/main/java/com/google/caliper/Param.java new file mode 100644 index 0000000..f4cdf45 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Param.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * To make your benchmark depend on a parameterized value, create a field with the name you want + * this parameter to be known by, and add this annotation. Caliper will inject a value for this + * field to each instance it creates. These values come from + * + *

    + *
  • The command line, if specified using {@code -Dname=value1,value2,value3} + *
  • Otherwise, the {@link #value()} list given in the annotation + *
  • Otherwise, Caliper looks for a static method named {@code paramName + "Values"} (for + * example, if the parameter field is {@code size}, it looks for {@code sizeValues()}). The + * method can return any subtype of {@link Iterable}. The contents of that iterable are used as + * the parameter values. + *
  • Otherwise, Caliper repeats the previous check looking for a static field instead + * of a method. + *
  • Otherwise, if the parameter type is either {@code boolean} or an {@code enum} type, Caliper + * assumes you want all possible values. + *
  • Finally, if none of the above match, Caliper will display an error and exit. + *
+ * + *

Caliper parameters are always strings, but can be converted to other types at the point of + * injection. If the type of the field this annotation is applied to is not {@link String}, then the + * type class must contain a static {@code fromString(String)}, {@code decode(String)} or {@code + * valueOf(String)} method that returns that type, or a constructor accepting only a {@code String}. + * + *

Caliper will test every possible combination of parameter values for your benchmark. For + * example, if you have two parameters, {@code -Dletter=a,b,c -Dnumber=1,2}, Caliper will construct + * six independent "scenarios" and perform measurement for each one. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface Param { + /** + * One or more default values, as strings, that this parameter should be given if none are + * specified on the command line. If values are specified on the command line, the defaults given + * here are all ignored. + */ + String[] value() default {}; +} diff --git a/caliper/src/main/java/com/google/caliper/Parameter.java b/caliper/src/main/java/com/google/caliper/Parameter.java new file mode 100644 index 0000000..c79bc86 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Parameter.java @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.annotations.VisibleForTesting; + +import java.lang.reflect.Field; +import java.lang.reflect.Member; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * A parameter in a {@link SimpleBenchmark}. + * + * @param the (possibly wrapped) type of the parameter field, such as {@link + * String} or {@link Integer} + */ +abstract class Parameter { + + private final Field field; + + private Parameter(Field field) { + this.field = field; + } + + /** + * Returns all parameters for the given class. + */ + public static Map> forClass(Class suiteClass) { + Map> parameters = new TreeMap>(); + for (Field field : suiteClass.getDeclaredFields()) { + if (field.isAnnotationPresent(Param.class)) { + field.setAccessible(true); + Parameter parameter = forField(suiteClass, field); + parameters.put(parameter.getName(), parameter); + } + } + return parameters; + } + + @VisibleForTesting + static Parameter forField( + Class suiteClass, final Field field) { + // First check for String values on the annotation itself + final Object[] defaults = field.getAnnotation(Param.class).value(); + if (defaults.length > 0) { + return new Parameter(field) { + @Override public Iterable values() throws Exception { + return Arrays.asList(defaults); + } + }; + // TODO: or should we continue so we can give an error/warning if params are also give in a + // method or field? + } + + Parameter result = null; + Type returnType = null; + Member member = null; + + // Now check for a fooValues() method + try { + final Method valuesMethod = suiteClass.getDeclaredMethod(field.getName() + "Values"); + if (!Modifier.isStatic(valuesMethod.getModifiers())) { + throw new ConfigurationException("Values method must be static " + member); + } + valuesMethod.setAccessible(true); + member = valuesMethod; + returnType = valuesMethod.getGenericReturnType(); + result = new Parameter(field) { + @SuppressWarnings("unchecked") // guarded below + @Override public Iterable values() throws Exception { + return (Iterable) valuesMethod.invoke(null); + } + }; + } catch (NoSuchMethodException ignored) { + } + + // Now check for a fooValues field + try { + final Field valuesField = suiteClass.getDeclaredField(field.getName() + "Values"); + if (!Modifier.isStatic(valuesField.getModifiers())) { + throw new ConfigurationException("Values field must be static " + member); + } + valuesField.setAccessible(true); + member = valuesField; + if (result != null) { + throw new ConfigurationException("Two values members defined for " + field); + } + returnType = valuesField.getGenericType(); + result = new Parameter(field) { + @SuppressWarnings("unchecked") // guarded below + @Override public Iterable values() throws Exception { + return (Iterable) valuesField.get(null); + } + }; + } catch (NoSuchFieldException ignored) { + } + + // If there isn't a values member but the parameter is an enum, we default + // to EnumSet.allOf. + if (member == null && field.getType().isEnum()) { + returnType = Collection.class; + result = new Parameter(field) { + // TODO: figure out the simplest way to make this compile and be green in IDEA too + @SuppressWarnings({"unchecked", "RawUseOfParameterizedType", "RedundantCast"}) + // guarded above + @Override public Iterable values() throws Exception { + Set set = EnumSet.allOf((Class) field.getType()); + return Collections.unmodifiableSet(set); + } + }; + } + + // If it's boolean, default to (true, false) + if (member == null && field.getType() == boolean.class) { + returnType = Collection.class; + result = new Parameter(field) { + @Override public Iterable values() throws Exception { + return Arrays.asList(Boolean.TRUE, Boolean.FALSE); + } + }; + } + + if (result == null) { + return new Parameter(field) { + @Override public Iterable values() { + // TODO: need tests to make sure this fails properly when no cmdline params given and + // works properly when they are given. Also, can we restructure the code so that we + // just throw here instead of later? + return Collections.emptySet(); + } + }; + } else if (!isValidReturnType(returnType)) { + throw new ConfigurationException("Invalid return type " + returnType + + " for values member " + member + "; must be Collection"); + } + return result; + } + + private static boolean isValidReturnType(Type type) { + if (type instanceof Class) { + return isIterableClass(type); + } + if (type instanceof ParameterizedType) { + return isIterableClass(((ParameterizedType) type).getRawType()); + } + return false; + } + + private static boolean isIterableClass(Type returnClass) { + return Iterable.class.isAssignableFrom((Class) returnClass); + } + + /** + * Sets the value of this property to the specified value for the given suite. + */ + public void set(Benchmark suite, Object value) throws Exception { + field.set(suite, value); + } + + /** + * Returns the available values of the property as specified by the suite. + */ + public abstract Iterable values() throws Exception; + + /** + * Returns the parameter's type, such as double.class. + */ + public Type getType() { + return field.getGenericType(); + } + + /** + * Returns the field's name. + */ + String getName() { + return field.getName(); + } +} diff --git a/caliper/src/main/java/com/google/caliper/Result.java b/caliper/src/main/java/com/google/caliper/Result.java new file mode 100644 index 0000000..c57f5a3 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Result.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +/** + * Represents an invocation of a benchmark, including the run itself, as well as the environment + * in which the run occurred. + */ +public final class Result { + private /*final*/ Run run; + private /*final*/ Environment environment; + + public Result(Run run, Environment environment) { + this.run = run; + this.environment = environment; + } + + public Run getRun() { + return run; + } + + public Environment getEnvironment() { + return environment; + } + + @Override public boolean equals(Object o) { + return o instanceof Result + && ((Result) o).run.equals(run) + && ((Result) o).environment.equals(environment); + } + + @Override public int hashCode() { + return run.hashCode() * 37 + environment.hashCode(); + } + + @Override public String toString() { + return run + "@" + environment; + } + + @SuppressWarnings("unused") + private Result() {} // for gson +} diff --git a/caliper/src/main/java/com/google/caliper/ResultsReader.java b/caliper/src/main/java/com/google/caliper/ResultsReader.java new file mode 100644 index 0000000..2396dd3 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/ResultsReader.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.gson.JsonParseException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; + +/** + * Helps with deserialization of results, given uncertainty about the format (xml or json) they + * are in. + */ +public final class ResultsReader { + public Result getResult(InputStream in) throws IOException { + // save input into a byte array since we may need to read it twice + byte[] postedData = readAllBytes(in); + Result result; + InputStreamReader baisJsonReader = new InputStreamReader(new ByteArrayInputStream(postedData)); + try { + result = Json.getGsonInstance().fromJson(baisJsonReader, Result.class); + } catch (JsonParseException e) { + // probably an old client is trying to send data, so try to parse it as XML instead. + ByteArrayInputStream baisXml = new ByteArrayInputStream(postedData); + try { + result = Xml.resultFromXml(baisXml); + } catch (Exception e2) { + throw new RuntimeException(e); + } finally { + baisXml.close(); + } + } finally { + baisJsonReader.close(); + } + return result; + } + + private byte[] readAllBytes(InputStream in) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + int read; + while ((read = in.read(buf)) != -1) { + baos.write(buf, 0, read); + } + return baos.toByteArray(); + } +} diff --git a/caliper/src/main/java/com/google/caliper/Run.java b/caliper/src/main/java/com/google/caliper/Run.java new file mode 100644 index 0000000..00f9350 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Run.java @@ -0,0 +1,93 @@ +/** + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.annotations.GwtCompatible; + +import java.io.Serializable; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The complete result of a benchmark suite run. + * + * WARNING: a JSON representation of this class is stored on the app engine server. If any changes + * are made to this class, a deserialization adapter must be written for this class to ensure + * backwards compatibility. + * + *

Gwt-safe. + */ +@SuppressWarnings("serial") +@GwtCompatible +public final class Run + implements Serializable /* for GWT Serialization */ { + + private /*final*/ Map measurements; + private /*final*/ String benchmarkName; + private /*final*/ long executedTimestamp; + + // TODO: add more run properties such as checksums of the executed code + + public Run(Map measurements, + String benchmarkName, Date executedTimestamp) { + if (benchmarkName == null || executedTimestamp == null) { + throw new NullPointerException(); + } + + this.measurements = new LinkedHashMap(measurements); + this.benchmarkName = benchmarkName; + this.executedTimestamp = executedTimestamp.getTime(); + } + + public Map getMeasurements() { + return measurements; + } + + public String getBenchmarkName() { + return benchmarkName; + } + + public Date getExecutedTimestamp() { + return new Date(executedTimestamp); + } + + @Override public boolean equals(Object o) { + if (o instanceof Run) { + Run that = (Run) o; + return measurements.equals(that.measurements) + && benchmarkName.equals(that.benchmarkName) + && executedTimestamp == that.executedTimestamp; + } + + return false; + } + + @Override public int hashCode() { + int result = measurements.hashCode(); + result = result * 37 + benchmarkName.hashCode(); + result = result * 37 + (int) ((executedTimestamp >> 32) ^ executedTimestamp); + return result; + } + + @Override public String toString() { + return measurements.toString(); + } + + @SuppressWarnings("unused") + private Run() {} // for GWT Serialization +} diff --git a/caliper/src/main/java/com/google/caliper/Runner.java b/caliper/src/main/java/com/google/caliper/Runner.java new file mode 100644 index 0000000..57715c9 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Runner.java @@ -0,0 +1,438 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.caliper.UserException.DisplayUsageException; +import com.google.caliper.UserException.ExceptionFromUserCodeException; +import com.google.caliper.util.InterleavedReader; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ObjectArrays; +import com.google.common.io.Closeables; +import com.google.gson.JsonObject; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintStream; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.URL; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TimeZone; +import java.util.regex.Pattern; + +/** + * Creates, executes and reports benchmark runs. + */ +public final class Runner { + + private static final FileFilter UPLOAD_FILE_FILTER = new FileFilter() { + @Override public boolean accept(File file) { + return file.getName().endsWith(".xml") || file.getName().endsWith(".json"); + } + }; + + private static final String FILE_NAME_DATE_FORMAT = "yyyy-MM-dd'T'HH-mm-ssZ"; + + private static final Splitter ARGUMENT_SPLITTER + = Splitter.on(Pattern.compile("\\s+")).omitEmptyStrings(); + + /** Command line arguments to the process */ + private Arguments arguments; + private ScenarioSelection scenarioSelection; + + private String createFileName(Result result) { + String timestamp = createTimestamp(); + return String.format("%s.%s.json", result.getRun().getBenchmarkName(), timestamp); + } + + private String createTimestamp() { + SimpleDateFormat dateFormat = new SimpleDateFormat(FILE_NAME_DATE_FORMAT, Locale.US); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + dateFormat.setLenient(true); + return dateFormat.format(new Date()); + } + + public void run(String... args) { + this.arguments = Arguments.parse(args); + File resultsUploadFile = arguments.getUploadResultsFile(); + if (resultsUploadFile != null) { + uploadResultsFileOrDir(resultsUploadFile); + return; + } + this.scenarioSelection = new ScenarioSelection(arguments); + if (arguments.getDebug()) { + debug(); + return; + } + Result result = runOutOfProcess(); + new ConsoleReport(result.getRun(), arguments).displayResults(); + boolean saveResultsLocally = arguments.getSaveResultsFile() != null; + try { + postResults(result); + } catch (Exception e) { + System.out.println(); + System.out.println(e); + saveResultsLocally = true; + } + + if (saveResultsLocally) { + saveResults(result); + } + } + + void uploadResultsFileOrDir(File resultsFileOrDir) { + try { + if (resultsFileOrDir.isDirectory()) { + for (File resultsFile : resultsFileOrDir.listFiles(UPLOAD_FILE_FILTER)) { + uploadResults(resultsFile); + } + } else { + uploadResults(resultsFileOrDir); + } + } catch (Exception e) { + throw new RuntimeException("uploading XML file failed", e); + } + } + + private void uploadResults(File resultsUploadFile) throws IOException { + System.out.println(); + System.out.println("Uploading " + resultsUploadFile.getCanonicalPath()); + InputStream inputStream = new FileInputStream(resultsUploadFile); + try { + Result result = new ResultsReader().getResult(inputStream); + postResults(result); + } finally { + inputStream.close(); + } + } + + private void saveResults(Result result) { + File resultsFile = arguments.getSaveResultsFile(); + File destinationFile; + if (resultsFile == null) { + File dir = new File("./caliper-results"); + dir.mkdirs(); + destinationFile = new File(dir, createFileName(result)); + } else if (resultsFile.exists() && resultsFile.isDirectory()) { + destinationFile = new File(resultsFile, createFileName(result)); + } else { + // assume this is a file + File parent = resultsFile.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + destinationFile = resultsFile; + } + + PrintStream filePrintStream; + try { + filePrintStream = new PrintStream(new FileOutputStream(destinationFile)); + } catch (FileNotFoundException e) { + throw new RuntimeException("can't open " + destinationFile, e); + } + String resultJson = Json.getGsonInstance().toJson(result); + try { + System.out.println(); + System.out.println("Writing results to " + destinationFile.getCanonicalPath()); + filePrintStream.print(resultJson); + } catch (Exception e) { + System.out.println(e); + System.out.println("Failed to write results to file, writing to standard out instead:"); + System.out.println(resultJson); + System.out.flush(); + } finally { + filePrintStream.close(); + } + } + + private void postResults(Result result) { + CaliperRc caliperrc = CaliperRc.INSTANCE; + String postUrl = caliperrc.getPostUrl(); + String apiKey = caliperrc.getApiKey(); + if (postUrl == null || apiKey == null) { + // TODO: probably nicer to show a message if only one is null + return; + } + + try { + URL url = new URL(postUrl + apiKey + "/" + result.getRun().getBenchmarkName()); + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(getProxy()); + urlConnection.setDoOutput(true); + String resultJson = Json.getGsonInstance().toJson(result); + urlConnection.getOutputStream().write(resultJson.getBytes()); + if (urlConnection.getResponseCode() == 200) { + System.out.println(""); + System.out.println("View current and previous benchmark results online:"); + BufferedReader in = new BufferedReader( + new InputStreamReader(urlConnection.getInputStream())); + System.out.println(" " + in.readLine()); + in.close(); + return; + } + + System.out.println("Posting to " + postUrl + " failed: " + + urlConnection.getResponseMessage()); + BufferedReader reader = new BufferedReader( + new InputStreamReader(urlConnection.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + reader.close(); + } catch (IOException e) { + throw new RuntimeException("Posting to " + postUrl + " failed.", e); + } + } + + private Proxy getProxy() { + String proxyAddress = CaliperRc.INSTANCE.getProxy(); + if (proxyAddress == null) { + return Proxy.NO_PROXY; + } + + String[] proxyHostAndPort = proxyAddress.trim().split(":"); + return new Proxy(Proxy.Type.HTTP, new InetSocketAddress( + proxyHostAndPort[0], Integer.parseInt(proxyHostAndPort[1]))); + } + + private ScenarioResult runScenario(Scenario scenario) { + MeasurementResult timeMeasurementResult = measure(scenario, MeasurementType.TIME); + MeasurementSet allocationMeasurements = null; + String allocationEventLog = null; + MeasurementSet memoryMeasurements = null; + String memoryEventLog = null; + if (arguments.getMeasureMemory()) { + MeasurementResult allocationsMeasurementResult = + measure(scenario, MeasurementType.INSTANCE); + allocationMeasurements = allocationsMeasurementResult.getMeasurements(); + allocationEventLog = allocationsMeasurementResult.getEventLog(); + MeasurementResult memoryMeasurementResult = + measure(scenario, MeasurementType.MEMORY); + memoryMeasurements = memoryMeasurementResult.getMeasurements(); + memoryEventLog = memoryMeasurementResult.getEventLog(); + } + + return new ScenarioResult(timeMeasurementResult.getMeasurements(), + timeMeasurementResult.getEventLog(), + allocationMeasurements, allocationEventLog, + memoryMeasurements, memoryEventLog); + } + + private static class MeasurementResult { + private final MeasurementSet measurements; + private final String eventLog; + + MeasurementResult(MeasurementSet measurements, String eventLog) { + this.measurements = measurements; + this.eventLog = eventLog; + } + + public MeasurementSet getMeasurements() { + return measurements; + } + + public String getEventLog() { + return eventLog; + } + } + + private MeasurementResult measure(Scenario scenario, MeasurementType type) { + Vm vm = new VmFactory().createVm(scenario); + // this must be done before starting the forked process on certain VMs + ProcessBuilder processBuilder = createCommand(scenario, vm, type) + .redirectErrorStream(true); + Process timeProcess; + try { + timeProcess = processBuilder.start(); + } catch (IOException e) { + throw new RuntimeException("failed to start subprocess", e); + } + + MeasurementSet measurementSet = null; + StringBuilder eventLog = new StringBuilder(); + InterleavedReader reader = null; + try { + reader = new InterleavedReader(arguments.getMarker(), + new InputStreamReader(timeProcess.getInputStream())); + Object o; + while ((o = reader.read()) != null) { + if (o instanceof String) { + eventLog.append(o); + } else if (measurementSet == null) { + JsonObject jsonObject = (JsonObject) o; + measurementSet = Json.measurementSetFromJson(jsonObject); + } else { + throw new RuntimeException("Unexpected value: " + o); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + Closeables.closeQuietly(reader); + timeProcess.destroy(); + } + + if (measurementSet == null) { + String message = "Failed to execute " + Joiner.on(" ").join(processBuilder.command()); + System.err.println(" " + message); + System.err.println(eventLog.toString()); + throw new ConfigurationException(message); + } + + return new MeasurementResult(measurementSet, eventLog.toString()); + } + + private ProcessBuilder createCommand(Scenario scenario, Vm vm, MeasurementType type) { + File workingDirectory = new File(System.getProperty("user.dir")); + + String classPath = System.getProperty("java.class.path"); + if (classPath == null || classPath.length() == 0) { + throw new IllegalStateException("java.class.path is undefined in " + System.getProperties()); + } + + ImmutableList.Builder vmArgs = ImmutableList.builder(); + vmArgs.addAll(ARGUMENT_SPLITTER.split(scenario.getVariables().get(Scenario.VM_KEY))); + if (type == MeasurementType.INSTANCE || type == MeasurementType.MEMORY) { + String allocationJarFile = System.getenv("ALLOCATION_JAR"); + vmArgs.add("-javaagent:" + allocationJarFile); + } + vmArgs.addAll(vm.getVmSpecificOptions(type, arguments)); + + Map vmParameters = scenario.getVariables( + scenarioSelection.getVmParameterNames()); + for (String vmParameter : vmParameters.values()) { + vmArgs.addAll(ARGUMENT_SPLITTER.split(vmParameter)); + } + + ImmutableList.Builder caliperArgs = ImmutableList.builder(); + caliperArgs.add("--warmupMillis").add(Long.toString(arguments.getWarmupMillis())); + caliperArgs.add("--runMillis").add(Long.toString(arguments.getRunMillis())); + caliperArgs.add("--measurementType").add(type.toString()); + caliperArgs.add("--marker").add(arguments.getMarker()); + + Map userParameters = scenario.getVariables( + scenarioSelection.getUserParameterNames()); + for (Entry entry : userParameters.entrySet()) { + caliperArgs.add("-D" + entry.getKey() + "=" + entry.getValue()); + } + caliperArgs.add(arguments.getSuiteClassName()); + + return vm.newProcessBuilder(workingDirectory, classPath, + vmArgs.build(), InProcessRunner.class.getName(), caliperArgs.build()); + } + + private void debug() { + try { + int debugReps = arguments.getDebugReps(); + InProcessRunner runner = new InProcessRunner(); + DebugMeasurer measurer = new DebugMeasurer(debugReps); + for (Scenario scenario : scenarioSelection.select()) { + System.out.println("running " + debugReps + " debug reps of " + scenario); + runner.run(scenarioSelection, scenario, measurer); + } + } catch (Exception e) { + throw new ExceptionFromUserCodeException(e); + } + } + + private Result runOutOfProcess() { + Date executedDate = new Date(); + ImmutableMap.Builder resultsBuilder = ImmutableMap.builder(); + + try { + List scenarios = scenarioSelection.select(); + + int i = 0; + for (Scenario scenario : scenarios) { + beforeMeasurement(i++, scenarios.size(), scenario); + ScenarioResult scenarioResult = runScenario(scenario); + afterMeasurement(arguments.getMeasureMemory(), scenarioResult); + resultsBuilder.put(scenario, scenarioResult); + } + System.out.println(); + + Environment environment = new EnvironmentGetter().getEnvironmentSnapshot(); + return new Result( + new Run(resultsBuilder.build(), arguments.getSuiteClassName(), executedDate), + environment); + } catch (Exception e) { + throw new ExceptionFromUserCodeException(e); + } + } + + private void beforeMeasurement(int index, int total, Scenario scenario) { + double percentDone = (double) index / total; + System.out.printf("%2.0f%% %s", percentDone * 100, scenario); + } + + private void afterMeasurement(boolean memoryMeasured, ScenarioResult scenarioResult) { + String memoryMeasurements = ""; + if (memoryMeasured) { + MeasurementSet instanceMeasurementSet = + scenarioResult.getMeasurementSet(MeasurementType.INSTANCE); + String instanceUnit = + ConsoleReport.UNIT_ORDERING.min(instanceMeasurementSet.getUnitNames().entrySet()).getKey(); + MeasurementSet memoryMeasurementSet = scenarioResult.getMeasurementSet(MeasurementType.MEMORY); + String memoryUnit = + ConsoleReport.UNIT_ORDERING.min(memoryMeasurementSet.getUnitNames().entrySet()).getKey(); + memoryMeasurements = String.format(", allocated %s%s for a total of %s%s", + Math.round(instanceMeasurementSet.medianUnits()), instanceUnit, + Math.round(memoryMeasurementSet.medianUnits()), memoryUnit); + } + + MeasurementSet timeMeasurementSet = scenarioResult.getMeasurementSet(MeasurementType.TIME); + String unit = + ConsoleReport.UNIT_ORDERING.min(timeMeasurementSet.getUnitNames().entrySet()).getKey(); + System.out.printf(" %.2f %s; \u03C3=%.2f %s @ %d trials%s%n", timeMeasurementSet.medianUnits(), + unit, timeMeasurementSet.standardDeviationUnits(), unit, + timeMeasurementSet.getMeasurements().size(), memoryMeasurements); + } + + public static void main(String[] args) { + try { + new Runner().run(args); + System.exit(0); // user code may have leave non-daemon threads behind! + } catch (DisplayUsageException e) { + e.display(); + System.exit(0); + } catch (UserException e) { + e.display(); + System.exit(1); + } + } + + @SuppressWarnings("unchecked") // temporary fakery + public static void main(Class suite, String[] args) { + main(ObjectArrays.concat(args, suite.getName())); + } +} diff --git a/caliper/src/main/java/com/google/caliper/Scenario.java b/caliper/src/main/java/com/google/caliper/Scenario.java new file mode 100644 index 0000000..bc026d8 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Scenario.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.annotations.GwtCompatible; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * A configured benchmark. + * + * WARNING: a JSON representation of this class is stored on the app engine server. If any changes + * are made to this class, a deserialization adapter must be written for this class to ensure + * backwards compatibility. + * + *

Gwt-safe. + */ +@SuppressWarnings("serial") +@GwtCompatible +public final class Scenario + implements Serializable /* for GWT */ { + + static final String VM_KEY = "vm"; + static final String TRIAL_KEY = "trial"; + + private /*final*/ Map variables; + + public Scenario(Map variables) { + this.variables = new LinkedHashMap(variables); + } + + public Map getVariables() { + return variables; + } + + /** + * Returns the named set of variables. + */ + public Map getVariables(Set names) { + Map result = new LinkedHashMap(variables); + result.keySet().retainAll(names); + if (!result.keySet().equals(names)) { + throw new IllegalArgumentException("Not all of " + names + " are in " + result.keySet()); + } + return result; + } + + @Override public boolean equals(Object o) { + return o instanceof Scenario + && ((Scenario) o).getVariables().equals(variables); + } + + @Override public int hashCode() { + return variables.hashCode(); + } + + @Override public String toString() { + return "Scenario" + variables; + } + + @SuppressWarnings("unused") + private Scenario() {} // for GWT +} diff --git a/caliper/src/main/java/com/google/caliper/ScenarioResult.java b/caliper/src/main/java/com/google/caliper/ScenarioResult.java new file mode 100644 index 0000000..1fa687d --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/ScenarioResult.java @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.annotations.GwtCompatible; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +/** + * Holds the results for a particular scenario, including timing measurements, memory use + * measurements, and event logs for both, recording significant events during measurement. + * + * WARNING: a JSON representation of this class is stored on the app engine server. If any changes + * are made to this class, a deserialization adapter must be written for this class to ensure + * backwards compatibility. + * + *

Gwt-safe. + */ +@SuppressWarnings({"serial", "FieldMayBeFinal"}) +@GwtCompatible +public final class ScenarioResult + implements Serializable /* for GWT Serialization */ { + + // want these to be EnumMaps, but that upsets GWT + private /*final*/ Map measurementSetMap + = new HashMap(); + private /*final*/ Map eventLogMap + = new HashMap(); + + public ScenarioResult(MeasurementSet timeMeasurementSet, + String timeEventLog, MeasurementSet instanceMeasurementSet, + String instanceEventLog, MeasurementSet memoryMeasurementSet, + String memoryEventLog) { + if (timeMeasurementSet != null) { + measurementSetMap.put(MeasurementType.TIME.toString(), timeMeasurementSet); + eventLogMap.put(MeasurementType.TIME.toString(), timeEventLog); + } + if (instanceMeasurementSet != null) { + measurementSetMap.put(MeasurementType.INSTANCE.toString(), instanceMeasurementSet); + eventLogMap.put(MeasurementType.INSTANCE.toString(), instanceEventLog); + } + if (memoryMeasurementSet != null) { + measurementSetMap.put(MeasurementType.MEMORY.toString(), memoryMeasurementSet); + eventLogMap.put(MeasurementType.MEMORY.toString(), memoryEventLog); + } + } + + public MeasurementSet getMeasurementSet(MeasurementType type) { + return measurementSetMap.get(type.toString()); + } + + public String getEventLog(MeasurementType type) { + return eventLogMap.get(type.toString()); + } + + @Override public boolean equals(Object o) { + return o instanceof ScenarioResult + && ((ScenarioResult) o).measurementSetMap.equals(measurementSetMap) + && ((ScenarioResult) o).eventLogMap.equals(eventLogMap); + } + + @Override public int hashCode() { + return measurementSetMap.hashCode() * 37 + eventLogMap.hashCode(); + } + + @Override public String toString() { + return "measurementSetMap: " + measurementSetMap + ", eventLogMap: " + eventLogMap; + } + + @SuppressWarnings("unused") + private ScenarioResult() {} // for GWT Serialization +} diff --git a/caliper/src/main/java/com/google/caliper/ScenarioSelection.java b/caliper/src/main/java/com/google/caliper/ScenarioSelection.java new file mode 100644 index 0000000..fcfead0 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/ScenarioSelection.java @@ -0,0 +1,253 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.caliper.UserException.AbstractBenchmarkException; +import com.google.caliper.UserException.DoesntImplementBenchmarkException; +import com.google.caliper.UserException.ExceptionFromUserCodeException; +import com.google.caliper.UserException.NoParameterlessConstructorException; +import com.google.caliper.UserException.NoSuchClassException; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Multimap; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +/** + * Figures out which scenarios to benchmark given a benchmark suite, set of user + * parameters, and set of user VMs. + */ +public final class ScenarioSelection { + + private final Set userVms; + private final Multimap vmParameters; + private final String suiteClassName; + + /** + * The user parameters specified on the command line. This may be a subset of + * the effective user parameters because parameters not specified here may get + * default values from the benchmark class. + */ + private final Multimap userParameterArguments; + + /** + * The actual user parameters we'll use to run in the benchmark. This contains + * the userParameterArguments plus the default user parameters. + */ + private Multimap userParameters; + + private final int trials; + private Benchmark suite; + + + public ScenarioSelection(Arguments arguments) { + this(arguments.getUserVms(), arguments.getVmParameters(), arguments.getSuiteClassName(), + arguments.getUserParameters(), arguments.getTrials()); + } + + public ScenarioSelection(Set userVms, Multimap vmParameters, + String suiteClassName, Multimap userParameterArguments, int trials) { + this.userVms = userVms; + this.vmParameters = vmParameters; + this.suiteClassName = suiteClassName; + this.userParameterArguments = userParameterArguments; + this.trials = trials; + } + + /** + * Returns the selected scenarios for this benchmark. + */ + public List select() { + prepareSuite(); + userParameters = computeUserParameters(); + return createScenarios(); + } + + /** + * Returns a normalized version of {@code scenario}, with information from {@code suite} + * assisting in correcting problems. + */ + public Scenario normalizeScenario(Scenario scenario) { + // This only applies to SimpleBenchmarks since they accept the special "benchmark" + // parameter. This is a special case because SimpleBenchmark is the most commonly + // used benchmark class. Have to do this horrible stuff since Benchmark API + // doesn't provide scenario-normalization (and it shouldn't), which SimpleBenchmark + // requires. + if (suite instanceof SimpleBenchmark) { + return ((SimpleBenchmark) suite).normalizeScenario(scenario); + } + + return scenario; + } + + public Set getUserParameterNames() { + if (userParameters == null) { + throw new IllegalStateException(); + } + return userParameters.keySet(); + } + + public Set getVmParameterNames() { + return vmParameters.keySet(); + } + + public ConfiguredBenchmark createBenchmark(Scenario scenario) { + return suite.createBenchmark(scenario.getVariables(getUserParameterNames())); + } + + private void prepareSuite() { + Class benchmarkClass; + try { + benchmarkClass = getClassByName(suiteClassName); + } catch (ExceptionInInitializerError e) { + throw new ExceptionFromUserCodeException(e.getCause()); + } catch (ClassNotFoundException ignored) { + throw new NoSuchClassException(suiteClassName); + } + + Object s; + try { + Constructor constructor = benchmarkClass.getDeclaredConstructor(); + constructor.setAccessible(true); + s = constructor.newInstance(); + } catch (InstantiationException ignore) { + throw new AbstractBenchmarkException(benchmarkClass); + } catch (NoSuchMethodException ignore) { + throw new NoParameterlessConstructorException(benchmarkClass); + } catch (IllegalAccessException impossible) { + throw new AssertionError(impossible); // shouldn't happen since we setAccessible(true) + } catch (InvocationTargetException e) { + throw new ExceptionFromUserCodeException(e.getCause()); + } + + if (s instanceof Benchmark) { + this.suite = (Benchmark) s; + } else { + throw new DoesntImplementBenchmarkException(benchmarkClass); + } + } + + private static Class getClassByName(String className) throws ClassNotFoundException { + try { + return Class.forName(className); + } catch (ClassNotFoundException ignored) { + // try replacing the last dot with a $, in case that helps + // example: tutorial.Tutorial.Benchmark1 becomes tutorial.Tutorial$Benchmark1 + // amusingly, the $ character means three different things in this one line alone + String newName = className.replaceFirst("\\.([^.]+)$", "\\$$1"); + return Class.forName(newName); + } + } + + private Multimap computeUserParameters() { + Multimap result = LinkedHashMultimap.create(); + for (String key : suite.parameterNames()) { + // first check if the user has specified values + Collection userValues = userParameterArguments.get(key); + if (!userValues.isEmpty()) { + result.putAll(key, userValues); + // TODO: type convert 'em to validate? + + } else { // otherwise use the default values from the suite + Set values = suite.parameterValues(key); + if (values.isEmpty()) { + throw new ConfigurationException(key + " has no values. " + + "Did you forget a -D" + key + "= command line argument?"); + } + result.putAll(key, values); + } + } + return result; + } + + /** + * Returns a complete set of scenarios with every combination of variables. + */ + private List createScenarios() { + List builders = new ArrayList(); + builders.add(new ScenarioBuilder()); + + Map> variables = new LinkedHashMap>(); + variables.put(Scenario.VM_KEY, userVms.isEmpty() ? VmFactory.defaultVms() : userVms); + variables.put(Scenario.TRIAL_KEY, newListOfSize(trials)); + variables.putAll(userParameters.asMap()); + variables.putAll(vmParameters.asMap()); + + for (Entry> entry : variables.entrySet()) { + Iterator values = entry.getValue().iterator(); + if (!values.hasNext()) { + throw new ConfigurationException("Not enough values for " + entry); + } + + String firstValue = values.next(); + for (ScenarioBuilder builder : builders) { + builder.variables.put(entry.getKey(), firstValue); + } + + // multiply the size of the specs by the number of alternate values + int size = builders.size(); + while (values.hasNext()) { + String alternate = values.next(); + for (int s = 0; s < size; s++) { + ScenarioBuilder copy = builders.get(s).copy(); + copy.variables.put(entry.getKey(), alternate); + builders.add(copy); + } + } + } + + List result = new ArrayList(); + for (ScenarioBuilder builder : builders) { + result.add(normalizeScenario(builder.build())); + } + + return result; + } + + /** + * Returns a list containing {@code count} distinct elements. + */ + private Collection newListOfSize(int count) { + List result = new ArrayList(); + for (int i = 0; i < count; i++) { + result.add(Integer.toString(i)); + } + return result; + } + + private static class ScenarioBuilder { + final Map variables = new LinkedHashMap(); + + ScenarioBuilder copy() { + ScenarioBuilder result = new ScenarioBuilder(); + result.variables.putAll(variables); + return result; + } + + public Scenario build() { + return new Scenario(variables); + } + } +} diff --git a/caliper/src/main/java/com/google/caliper/SimpleBenchmark.java b/caliper/src/main/java/com/google/caliper/SimpleBenchmark.java new file mode 100644 index 0000000..01fc3a8 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/SimpleBenchmark.java @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.caliper.UserException.ExceptionFromUserCodeException; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * A convenience class for implementing benchmarks in plain code. + * Implementing classes must have a no-arguments constructor. + * + *

Benchmarks

+ * The benchmarks of a suite are defined by . They may be + * static. They are not permitted to take parameters . . .. + * + *

Parameters

+ * See the {@link Param} documentation to learn about parameters. + */ +public abstract class SimpleBenchmark + implements Benchmark { + private static final Class[] ARGUMENT_TYPES = { int.class }; + + private final Map> parameters; + private final Map methods; + + protected SimpleBenchmark() { + parameters = Parameter.forClass(getClass()); + methods = createTimedMethods(); + + if (methods.isEmpty()) { + throw new ConfigurationException( + "No benchmarks defined in " + getClass().getName()); + } + } + + protected void setUp() throws Exception {} + + protected void tearDown() throws Exception {} + + @Override public Set parameterNames() { + return ImmutableSet.builder() + .add("benchmark") + .addAll(parameters.keySet()) + .build(); + } + + @Override public Set parameterValues(String parameterName) { + if ("benchmark".equals(parameterName)) { + return methods.keySet(); + } + + Parameter parameter = parameters.get(parameterName); + if (parameter == null) { + throw new IllegalArgumentException(); + } + try { + Iterable values = parameter.values(); + + ImmutableSet.Builder result = ImmutableSet.builder(); + for (Object value : values) { + result.add(String.valueOf(value)); + } + return result.build(); + } catch (Exception e) { + throw new ExceptionFromUserCodeException(e); + } + } + + @Override public ConfiguredBenchmark createBenchmark(Map parameterValues) { + if (!parameterNames().equals(parameterValues.keySet())) { + throw new IllegalArgumentException("Invalid parameters specified. Expected " + + parameterNames() + " but was " + parameterValues.keySet()); + } + + String methodName = parameterValues.get("benchmark"); + final Method method = methods.get(methodName); + if (method == null) { + throw new IllegalArgumentException("Invalid parameters specified. \"time" + methodName + "\" " + + "is not a method of this benchmark."); + } + + try { + @SuppressWarnings({"ClassNewInstance"}) // can throw any Exception, so we catch all Exceptions + final SimpleBenchmark copyOfSelf = getClass().newInstance(); + + for (Map.Entry entry : parameterValues.entrySet()) { + String parameterName = entry.getKey(); + if ("benchmark".equals(parameterName)) { + continue; + } + + Parameter parameter = parameters.get(parameterName); + Object value = TypeConverter.fromString(entry.getValue(), parameter.getType()); + parameter.set(copyOfSelf, value); + } + copyOfSelf.setUp(); + + return new ConfiguredBenchmark(copyOfSelf) { + @Override public Object run(int reps) throws Exception { + try { + return method.invoke(copyOfSelf, reps); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Exception) { + throw (Exception) cause; + } else if (cause instanceof Error) { + throw (Error) cause; + } else { + throw e; + } + } + } + + @Override public void close() throws Exception { + copyOfSelf.tearDown(); + } + }; + } catch (Exception e) { + throw new ExceptionFromUserCodeException(e); + } + } + + public Scenario normalizeScenario(Scenario scenario) { + Map variables = + new LinkedHashMap(scenario.getVariables()); + // Make sure the scenario contains method names without the prefixed "time". If + // it has "time" prefixed, then remove it. Also check whether the user has + // accidentally put a lower cased letter first, and fix it if necessary. + String benchmark = variables.get("benchmark"); + Map timedMethods = createTimedMethods(); + if (timedMethods.get(benchmark) == null) { + // try to upper case first character + char[] benchmarkChars = benchmark.toCharArray(); + benchmarkChars[0] = Character.toUpperCase(benchmarkChars[0]); + String upperCasedBenchmark = String.valueOf(benchmarkChars); + if (timedMethods.get(upperCasedBenchmark) != null) { + variables.put("benchmark", upperCasedBenchmark); + } else if (benchmark.startsWith("time")) { + variables.put("benchmark", benchmark.substring(4)); + } + } + return new Scenario(variables); + } + + /** + * Returns a spec for each benchmark defined in the specified class. The + * returned specs have no parameter values; those must be added separately. + */ + private Map createTimedMethods() { + ImmutableMap.Builder result = ImmutableMap.builder(); + for (Method method : getClass().getDeclaredMethods()) { + int modifiers = method.getModifiers(); + if (!method.getName().startsWith("time")) { + continue; + } + + if (!Modifier.isPublic(modifiers) + || Modifier.isStatic(modifiers) + || Modifier.isAbstract(modifiers) + || !Arrays.equals(method.getParameterTypes(), ARGUMENT_TYPES)) { + throw new ConfigurationException("Timed methods must be public, " + + "non-static, non-abstract and take a single int parameter. " + + "But " + method + " violates these requirements."); + } + + result.put(method.getName().substring(4), method); + } + + return result.build(); + } + + @Override public Map getTimeUnitNames() { + return ImmutableMap.of("ns", 1, + "us", 1000, + "ms", 1000000, + "s", 1000000000); + } + + @Override public double nanosToUnits(double nanos) { + return nanos; + } + + @Override public Map getInstanceUnitNames() { + return ImmutableMap.of(" instances", 1, + "K instances", 1000, + "M instances", 1000000, + "B instances", 1000000000); + } + + @Override public double instancesToUnits(long instances) { + return instances; + } + + @Override public Map getMemoryUnitNames() { + return ImmutableMap.of("B", 1, + "KiB", 1024, + "MiB", 1048576, + "GiB", 1073741824); + } + + @Override public double bytesToUnits(long bytes) { + return bytes; + } +} diff --git a/caliper/src/main/java/com/google/caliper/StandardVm.java b/caliper/src/main/java/com/google/caliper/StandardVm.java new file mode 100644 index 0000000..3366eb1 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/StandardVm.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.ArrayList; +import java.util.List; + +final class StandardVm extends Vm { + + @Override public List getVmSpecificOptions(MeasurementType type, Arguments arguments) { + if (!arguments.getCaptureVmLog()) { + return ImmutableList.of(); + } + + List result = new ArrayList(); + result.add("-verbose:gc"); + result.add("-Xbatch"); + result.add("-XX:+UseSerialGC"); + if (type == MeasurementType.TIME) { + return Lists.newArrayList("-XX:+PrintCompilation"); + } + + return result; + } + + public static String defaultVmName() { + return "java"; + } +} diff --git a/caliper/src/main/java/com/google/caliper/TimeMeasurer.java b/caliper/src/main/java/com/google/caliper/TimeMeasurer.java new file mode 100644 index 0000000..2925d2d --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/TimeMeasurer.java @@ -0,0 +1,180 @@ +/** + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.caliper.UserException.DoesNotScaleLinearlyException; +import com.google.caliper.UserException.RuntimeOutOfRangeException; +import com.google.common.base.Supplier; + +/** + * Measure's the benchmark's per-trial execution time. + */ +class TimeMeasurer extends Measurer { + + private final long warmupNanos; + private final long runNanos; + + /** + * If the standard deviation of our measurements is within this tolerance, we + * won't bother to perform additional measurements. + */ + private static final double SHORT_CIRCUIT_TOLERANCE = 0.01; + + private static final int MAX_TRIALS = 10; + + TimeMeasurer(long warmupMillis, long runMillis) { + checkArgument(warmupMillis > 50); + checkArgument(runMillis > 50); + + this.warmupNanos = warmupMillis * 1000000; + this.runNanos = runMillis * 1000000; + } + + private double warmUp(Supplier testSupplier) throws Exception { + long elapsedNanos = 0; + long netReps = 0; + int reps = 1; + boolean definitelyScalesLinearly = false; + + /* + * Run progressively more reps at a time until we cross our warmup + * threshold. This way any just-in-time compiler will be comfortable running + * multiple iterations of our measurement method. + */ + log("[starting warmup]"); + while (elapsedNanos < warmupNanos) { + long nanos = measureReps(testSupplier.get(), reps); + elapsedNanos += nanos; + + netReps += reps; + reps *= 2; + + // if reps overflowed, that's suspicious! Check that it time scales with reps + if (reps <= 0) { + if (!definitelyScalesLinearly) { + checkScalesLinearly(testSupplier); + definitelyScalesLinearly = true; + } + reps = Integer.MAX_VALUE; + } + } + log("[ending warmup]"); + + double nanosPerExecution = (double) elapsedNanos / netReps; + double lowerBound = 0.1; + double upperBound = 10000000000.0; + if (!(lowerBound <= nanosPerExecution && nanosPerExecution <= upperBound)) { + throw new RuntimeOutOfRangeException(nanosPerExecution, lowerBound, upperBound); + } + + return nanosPerExecution; + } + + /** + * Doing half as much work shouldn't take much more than half as much time. If + * it does we have a broken benchmark! + */ + private void checkScalesLinearly(Supplier testSupplier) throws Exception { + double half = measureReps(testSupplier.get(), Integer.MAX_VALUE / 2); + double one = measureReps(testSupplier.get(), Integer.MAX_VALUE); + if (half / one > 0.75) { + throw new DoesNotScaleLinearlyException(); + } + } + + /** + * Measure the nanos per rep for the given test. This code uses an interesting + * strategy to measure the runtime to minimize execution time when execution + * time is consistent. + *
    + *
  1. 1.0x {@code runMillis} trial is run. + *
  2. 0.5x {@code runMillis} trial is run. + *
  3. 1.5x {@code runMillis} trial is run. + *
  4. At this point, the standard deviation of these trials is computed. If + * it is within the threshold, the result is returned. + *
  5. Otherwise trials continue to be executed until either the threshold + * is satisfied or the maximum number of runs have been executed. + *
+ * + * @param testSupplier provides instances of the code under test. A new test + * is created for each iteration because some benchmarks' performance + * depends on which memory was allocated. See SetContainsBenchmark for an + * example. + */ + @Override public MeasurementSet run(Supplier testSupplier) + throws Exception { + double estimatedNanosPerRep = warmUp(testSupplier); + + log("[measuring nanos per rep with scale 1.00]"); + Measurement measurement100 = measure(testSupplier, 1.00, estimatedNanosPerRep); + log("[measuring nanos per rep with scale 0.50]"); + Measurement measurement050 = measure(testSupplier, 0.50, measurement100.getRaw()); + log("[measuring nanos per rep with scale 1.50]"); + Measurement measurement150 = measure(testSupplier, 1.50, measurement100.getRaw()); + MeasurementSet measurementSet = + new MeasurementSet(measurement100, measurement050, measurement150); + + for (int i = 3; i < MAX_TRIALS; i++) { + double threshold = SHORT_CIRCUIT_TOLERANCE * measurementSet.meanRaw(); + if (measurementSet.standardDeviationRaw() < threshold) { + return measurementSet; + } + + log("[performing additional measurement with scale 1.00]"); + Measurement measurement = measure(testSupplier, 1.00, measurement100.getRaw()); + measurementSet = measurementSet.plusMeasurement(measurement); + } + + return measurementSet; + } + + /** + * Runs the test method for approximately {@code runNanos * durationScale} + * nanos and returns a Measurement of the nanos per rep and units per rep. + */ + private Measurement measure(Supplier testSupplier, + double durationScale, double estimatedNanosPerRep) throws Exception { + int reps = (int) (durationScale * runNanos / estimatedNanosPerRep); + if (reps == 0) { + reps = 1; + } + + log("[running trial with " + reps + " reps]"); + ConfiguredBenchmark benchmark = testSupplier.get(); + long elapsedTime = measureReps(benchmark, reps); + double nanosPerRep = elapsedTime / (double) reps; + log(String.format("[took %.2f nanoseconds per rep]", nanosPerRep)); + return new Measurement(benchmark.timeUnitNames(), nanosPerRep, + benchmark.nanosToUnits(nanosPerRep)); + } + + /** + * Returns the total nanos to run {@code reps}. + */ + private long measureReps(ConfiguredBenchmark benchmark, int reps) throws Exception { + prepareForTest(); + log(LogConstants.MEASURED_SECTION_STARTING); + long startNanos = System.nanoTime(); + benchmark.run(reps); + long endNanos = System.nanoTime(); + log(LogConstants.MEASURED_SECTION_DONE); + benchmark.close(); + return endNanos - startNanos; + } +} diff --git a/caliper/src/main/java/com/google/caliper/TypeConverter.java b/caliper/src/main/java/com/google/caliper/TypeConverter.java new file mode 100644 index 0000000..c3268e0 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/TypeConverter.java @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableMap; + +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.Map; + +/** + * Convert objects to and from Strings. + */ +final class TypeConverter { + private TypeConverter() {} + + public static Object fromString(String value, Type type) { + if (type == String.class) { + return value; + } + + Class c = wrap((Class) type); + try { + Method m = c.getMethod("valueOf", String.class); + m.setAccessible(true); // to permit inner enums, etc. + return m.invoke(null, value); + } catch (Exception e) { + throw new UnsupportedOperationException( + "Cannot convert " + value + " of type " + type, e); + } + } + + // safe because both Long.class and long.class are of type Class + @SuppressWarnings("unchecked") + private static Class wrap(Class c) { + return c.isPrimitive() ? (Class) PRIMITIVES_TO_WRAPPERS.get(c) : c; + } + + private static final Map, Class> PRIMITIVES_TO_WRAPPERS + = new ImmutableMap.Builder, Class>() + .put(boolean.class, Boolean.class) + .put(byte.class, Byte.class) + .put(char.class, Character.class) + .put(double.class, Double.class) + .put(float.class, Float.class) + .put(int.class, Integer.class) + .put(long.class, Long.class) + .put(short.class, Short.class) + .put(void.class, Void.class) + .build(); +} diff --git a/caliper/src/main/java/com/google/caliper/UploadResults.java b/caliper/src/main/java/com/google/caliper/UploadResults.java new file mode 100644 index 0000000..7dae7af --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/UploadResults.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import java.io.File; + +/** + * Usage: UploadResults + */ +public class UploadResults { + public static void main(String[] args) { + new Runner().uploadResultsFileOrDir(new File(args[0])); + } +} diff --git a/caliper/src/main/java/com/google/caliper/UserException.java b/caliper/src/main/java/com/google/caliper/UserException.java new file mode 100644 index 0000000..a4535a7 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/UserException.java @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import java.util.Arrays; +import java.util.Set; + +/** + * Signifies a problem that should be explained in user-friendly terms on the command line, without + * a confusing stack trace, and optionally followed by a usage summary. + */ +@SuppressWarnings("serial") // never going to serialize these... right? +public abstract class UserException extends RuntimeException { + protected UserException(String error) { + super(error); + } + + public abstract void display(); + + // - - - - + + public abstract static class ErrorInUsageException extends UserException { + protected ErrorInUsageException(String error) { + super(error); + } + + @Override public void display() { + String message = getMessage(); + if (message != null) { + System.err.println("Error: " + message); + } + Arguments.printUsage(); + } + } + + public abstract static class ErrorInUserCodeException extends UserException { + private final String remedy; + + protected ErrorInUserCodeException(String error, String remedy) { + super(error); + this.remedy = remedy; + } + + @Override public void display() { + System.err.println("Error: " + getMessage()); + System.err.println("Typical Remedy: " + remedy); + } + } + + // - - - - + + // Not technically an error, but works nicely this way anyway + public static class DisplayUsageException extends ErrorInUsageException { + public DisplayUsageException() { + super(null); + } + } + + public static class IncompatibleArgumentsException extends ErrorInUsageException { + public IncompatibleArgumentsException(String arg) { + super("Some arguments passed in are incompatible with: " + arg); + } + } + + public static class UnrecognizedOptionException extends ErrorInUsageException { + public UnrecognizedOptionException(String arg) { + super("Argument not recognized: " + arg); + } + } + + public static class NoBenchmarkClassException extends ErrorInUsageException { + public NoBenchmarkClassException() { + super("No benchmark class specified."); + } + } + + public static class MultipleBenchmarkClassesException extends ErrorInUsageException { + public MultipleBenchmarkClassesException(String a, String b) { + super("Multiple benchmark classes specified: " + Arrays.asList(a, b)); + } + } + + public static class MalformedParameterException extends ErrorInUsageException { + public MalformedParameterException(String arg) { + super("Malformed parameter: " + arg); + } + } + + public static class DuplicateParameterException extends ErrorInUsageException { + public DuplicateParameterException(String arg) { + super("Duplicate parameter: " + arg); + } + public DuplicateParameterException(Set arg) { + super("Duplicate parameters: " + arg); + } + } + + public static class InvalidParameterValueException extends ErrorInUsageException { + public InvalidParameterValueException(String arg, String value) { + super("Invalid value \"" + value + "\" for parameter: " + arg); + } + } + + public static class InvalidTrialsException extends ErrorInUsageException { + public InvalidTrialsException(String arg) { + super("Invalid trials: " + arg); + } + } + + public static class CantCustomizeInProcessVmException extends ErrorInUsageException { + public CantCustomizeInProcessVmException() { + super("Can't customize VM when running in process."); + } + } + + public static class NoSuchClassException extends ErrorInUsageException { + public NoSuchClassException(String name) { + super("No class named [" + name + "] was found (check CLASSPATH)."); + } + } + + public static class RuntimeOutOfRangeException extends ErrorInUsageException { + public RuntimeOutOfRangeException( + double nanosPerExecution, double lowerBound, double upperBound) { + super("Runtime " + nanosPerExecution + "ns/rep out of range " + + lowerBound + "-" + upperBound); + } + } + + public static class DoesNotScaleLinearlyException extends ErrorInUsageException { + public DoesNotScaleLinearlyException() { + super("Doing 2x as much work didn't take 2x as much time! " + + "Is the JIT optimizing away the body of your benchmark?"); + } + } + + public static class NonConstantMemoryUsage extends ErrorInUsageException { + public NonConstantMemoryUsage() { + super("Not all reps of the inner loop allocate the same number of times! " + + "The reps loop should use a constant number of allocations. " + + "Are you using the value of reps inside the loop?"); + } + } + + public static class AbstractBenchmarkException extends ErrorInUserCodeException { + public AbstractBenchmarkException(Class specifiedClass) { + super("Class [" + specifiedClass.getName() + "] is abstract.", "Specify a concrete class."); + } + } + + public static class NoParameterlessConstructorException extends ErrorInUserCodeException { + public NoParameterlessConstructorException(Class specifiedClass) { + super("Class [" + specifiedClass.getName() + "] has no parameterless constructor.", + "Remove all constructors or add a parameterless constructor."); + } + } + + public static class DoesntImplementBenchmarkException extends ErrorInUserCodeException { + public DoesntImplementBenchmarkException(Class specifiedClass) { + super("Class [" + specifiedClass + "] does not implement the " + Benchmark.class.getName() + + " interface.", "Add 'extends " + SimpleBenchmark.class + "' to the class declaration."); + } + } + + public static class InvalidDebugRepsException extends ErrorInUsageException { + public InvalidDebugRepsException(String arg) { + super("Invalid debug reps: " + arg); + } + } + + // TODO: should remove the caliper stack frames.... + public static class ExceptionFromUserCodeException extends UserException { + public ExceptionFromUserCodeException(Throwable t) { + super("An exception was thrown from the benchmark code."); + initCause(t); + } + @Override public void display() { + System.err.println(getMessage()); + getCause().printStackTrace(System.err); + } + } +} diff --git a/caliper/src/main/java/com/google/caliper/Vm.java b/caliper/src/main/java/com/google/caliper/Vm.java new file mode 100644 index 0000000..1cc45a9 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Vm.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableList; + +import java.io.File; +import java.util.List; + +class Vm { + public List getVmSpecificOptions(MeasurementType type, Arguments arguments) { + return ImmutableList.of(); + } + + /** + * Returns a process builder to run this VM. + * + * @param vmArgs the path to the VM followed by VM arguments. + * @param applicationArgs arguments to the target process + */ + public ProcessBuilder newProcessBuilder(File workingDirectory, String classPath, + ImmutableList vmArgs, String className, ImmutableList applicationArgs) { + ProcessBuilder result = new ProcessBuilder(); + result.directory(workingDirectory); + result.command().addAll(vmArgs); + result.command().add("-cp"); + result.command().add(classPath); + result.command().add(className); + result.command().addAll(applicationArgs); + return result; + } +} diff --git a/caliper/src/main/java/com/google/caliper/VmFactory.java b/caliper/src/main/java/com/google/caliper/VmFactory.java new file mode 100644 index 0000000..408e34c --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/VmFactory.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableSet; + +import java.util.Arrays; +import java.util.List; + +public final class VmFactory { + public static ImmutableSet defaultVms() { + String vmName = DalvikVm.isDalvikVm() + ? DalvikVm.vmName() + : StandardVm.defaultVmName(); + return ImmutableSet.of(vmName); + } + + public Vm createVm(Scenario scenario) { + List vmList = Arrays.asList(scenario.getVariables().get(Scenario.VM_KEY).split("\\s+")); + Vm vm = null; + if (!vmList.isEmpty()) { + if (vmList.get(0).endsWith("app_process")) { + vm = new DalvikVm(); + } else if (vmList.get(0).endsWith("java")) { + vm = new StandardVm(); + } + } + if (vm == null) { + vm = new Vm(); + } + return vm; + } +} diff --git a/caliper/src/main/java/com/google/caliper/Xml.java b/caliper/src/main/java/com/google/caliper/Xml.java new file mode 100644 index 0000000..2aeebda --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/Xml.java @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableMap; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; + +import java.io.InputStream; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; + +import javax.xml.parsers.DocumentBuilderFactory; + +/** + * This exists for backwards compatibility with old data, which is stored in XML format. + * All new data is stored in JSON. + */ +public final class Xml { + private static final String DATE_FORMAT_STRING = "yyyy-MM-dd'T'HH:mm:ssz"; + private static final String ENVIRONMENT_ELEMENT_NAME = "environment"; + private static final String RESULT_ELEMENT_NAME = "result"; + private static final String RUN_ELEMENT_NAME = "run"; + private static final String BENCHMARK_ATTRIBUTE = "benchmark"; + private static final String EXECUTED_TIMESTAMP_ATTRIBUTE = "executedTimestamp"; + private static final String OLD_SCENARIO_ELEMENT_NAME = "scenario"; + // for backwards compatibility, use a different name + private static final String SCENARIO_ELEMENT_NAME = "newScenario"; + private static final String MEASUREMENTS_ELEMENT_NAME = "measurements"; + private static final String TIME_EVENT_LOG_ELEMENT_NAME = "eventLog"; + + private static Result readResultElement(Element element) throws Exception { + Environment environment = null; + Run run = null; + for (Node topLevelNode : XmlUtils.childrenOf(element)) { + if (topLevelNode.getNodeName().equals(ENVIRONMENT_ELEMENT_NAME)) { + Element environmentElement = (Element) topLevelNode; + environment = readEnvironmentElement(environmentElement); + } else if (topLevelNode.getNodeName().equals(RUN_ELEMENT_NAME)) { + run = readRunElement((Element) topLevelNode); + } else { + throw new RuntimeException("illegal node name: " + topLevelNode.getNodeName()); + } + } + + if (environment == null || run == null) { + throw new RuntimeException("missing environment or run elements"); + } + + return new Result(run, environment); + } + + private static Environment readEnvironmentElement(Element element) { + return new Environment(XmlUtils.attributesOf(element)); + } + + private static Run readRunElement(Element element) throws Exception { + String benchmarkName = element.getAttribute(BENCHMARK_ATTRIBUTE); + String executedDateString = element.getAttribute(EXECUTED_TIMESTAMP_ATTRIBUTE); + Date executedDate = new SimpleDateFormat(DATE_FORMAT_STRING, Locale.US) + .parse(executedDateString); + + ImmutableMap.Builder measurementsBuilder = ImmutableMap.builder(); + for (Node scenarioNode : XmlUtils.childrenOf(element)) { + Element scenarioElement = (Element) scenarioNode; + Scenario scenario = new Scenario(XmlUtils.attributesOf(scenarioElement)); + ScenarioResult scenarioResult; + + // for backwards compatibility with older runs + if (scenarioNode.getNodeName().equals(OLD_SCENARIO_ELEMENT_NAME)) { + MeasurementSet measurement = + Json.measurementSetFromJson(scenarioElement.getTextContent()); + scenarioResult = new ScenarioResult(measurement, "", + null, null, null, null); + } else if (scenarioNode.getNodeName().equals(SCENARIO_ELEMENT_NAME)) { + MeasurementSet timeMeasurementSet = null; + String eventLog = null; + for (Node node : XmlUtils.childrenOf(scenarioElement)) { + if (node.getNodeName().equals(MEASUREMENTS_ELEMENT_NAME)) { + timeMeasurementSet = Json.measurementSetFromJson(node.getTextContent()); + } else if (node.getNodeName().equals(TIME_EVENT_LOG_ELEMENT_NAME)) { + eventLog = node.getTextContent(); + } else { + throw new RuntimeException("illegal node name: " + node.getNodeName()); + } + } + if (timeMeasurementSet == null || eventLog == null) { + throw new RuntimeException("missing node \"" + MEASUREMENTS_ELEMENT_NAME + "\" or \"" + + TIME_EVENT_LOG_ELEMENT_NAME + "\""); + } + // "new Measurement[0]" used instead of empty varargs argument since MeasurementSet has + // an empty private constructor. + scenarioResult = new ScenarioResult(timeMeasurementSet, eventLog, + null, null, null, null); + } else { + throw new RuntimeException("illegal node name: " + scenarioNode.getNodeName()); + } + + measurementsBuilder.put(scenario, scenarioResult); + } + + return new Run(measurementsBuilder.build(), benchmarkName, executedDate); + } + + /** + * Creates a result by decoding XML from the specified stream. The XML should + * be consistent with the format emitted by the now deleted runToXml(Run, OutputStream). + */ + public static Run runFromXml(InputStream in) { + try { + Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(in); + return readRunElement(document.getDocumentElement()); + } catch (Exception e) { + throw new IllegalStateException("Malformed XML document", e); + } + } + + /** + * Creates an environment by decoding XML from the specified stream. The XML should + * be consistent with the format emitted by the now deleted + * environmentToXml(Environment, OutputStream). + */ + public static Environment environmentFromXml(InputStream in) { + try { + Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(in); + Element environmentElement = document.getDocumentElement(); + return readEnvironmentElement(environmentElement); + } catch (Exception e) { + throw new IllegalStateException("Malformed XML document", e); + } + } + + public static Result resultFromXml(InputStream in) { + try { + Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(in); + return readResultElement(document.getDocumentElement()); + } catch (Exception e) { + throw new IllegalStateException("Malformed XML document", e); + } + } + + private Xml() {} +} diff --git a/caliper/src/main/java/com/google/caliper/XmlUtils.java b/caliper/src/main/java/com/google/caliper/XmlUtils.java new file mode 100644 index 0000000..458e6fa --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/XmlUtils.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import org.w3c.dom.Attr; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +public final class XmlUtils { + public static ImmutableList childrenOf(Node node) { + NodeList children = node.getChildNodes(); + ImmutableList.Builder result = ImmutableList.builder(); + for (int i = 0, size = children.getLength(); i < size; i++) { + result.add(children.item(i)); + } + return result.build(); + } + + public static ImmutableMap attributesOf(Element element) { + NamedNodeMap map = element.getAttributes(); + ImmutableMap.Builder result = ImmutableMap.builder(); + for (int i = 0, size = map.getLength(); i < size; i++) { + Attr attr = (Attr) map.item(i); + result.put(attr.getName(), attr.getValue()); + } + return result.build(); + } + + private XmlUtils() {} +} diff --git a/caliper/src/main/java/com/google/caliper/util/InterleavedReader.java b/caliper/src/main/java/com/google/caliper/util/InterleavedReader.java new file mode 100644 index 0000000..40293bf --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/util/InterleavedReader.java @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper.util; + +import com.google.gson.JsonParser; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.IOException; +import java.io.Reader; + +/** + * Reads a stream containing inline JSON objects. Each JSON object is prefixed + * by a marker string and suffixed by a newline character. + */ +public final class InterleavedReader implements Closeable { + + /** + * The length of the scratch buffer to search for markers in. Also acts as an + * upper bound on the length of returned strings. Not used as an I/O buffer. + */ + private static final int BUFFER_LENGTH = 80; + + private final String marker; + private final BufferedReader reader; + private final JsonParser jsonParser = new JsonParser(); + + public static final String DEFAULT_MARKER = "//ZxJ/"; + + public InterleavedReader(Reader reader) { + this(DEFAULT_MARKER, reader); + } + + public InterleavedReader(String marker, Reader reader) { + if (marker.length() > BUFFER_LENGTH) { + throw new IllegalArgumentException("marker.length() > BUFFER_LENGTH"); + } + this.marker = marker; + this.reader = reader instanceof BufferedReader + ? (BufferedReader) reader + : new BufferedReader(reader); + } + + /** + * Returns the next value in the stream: either a String, a JsonElement, or + * null to indicate the end of the stream. Callers should use instanceof to + * inspect the return type. + */ + public Object read() throws IOException { + char[] buffer = new char[BUFFER_LENGTH]; + reader.mark(BUFFER_LENGTH); + int count = 0; + int textEnd; + + while (true) { + int r = reader.read(buffer, count, buffer.length - count); + + if (r == -1) { + // the input is exhausted; return the remaining characters + textEnd = count; + break; + } + + count += r; + int possibleMarker = findPossibleMarker(buffer, count); + + if (possibleMarker != 0) { + // return the characters that precede the marker + textEnd = possibleMarker; + break; + } + + if (count < marker.length()) { + // the buffer contains only the prefix of a marker so we must read more + continue; + } + + // we've read a marker so return the value that follows + reader.reset(); + String json = reader.readLine().substring(marker.length()); + return jsonParser.parse(json); + } + + if (count == 0) { + return null; + } + + // return characters + reader.reset(); + count = reader.read(buffer, 0, textEnd); + return new String(buffer, 0, count); + } + + @Override public void close() throws IOException { + reader.close(); + } + + /** + * Returns the index of marker in {@code chars}, stopping at {@code limit}. + * Should the chars end with a prefix of marker, the offset of that prefix + * is returned. + */ + int findPossibleMarker(char[] chars, int limit) { + search: + for (int i = 0; true; i++) { + for (int m = 0; m < marker.length() && i + m < limit; m++) { + if (chars[i + m] != marker.charAt(m)) { + continue search; + } + } + return i; + } + } +} diff --git a/caliper/src/main/java/com/google/caliper/util/LinearTranslation.java b/caliper/src/main/java/com/google/caliper/util/LinearTranslation.java new file mode 100644 index 0000000..2e52b06 --- /dev/null +++ b/caliper/src/main/java/com/google/caliper/util/LinearTranslation.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper.util; + +import com.google.common.annotations.GwtCompatible; + +@GwtCompatible +public class LinearTranslation { + // y = mx + b + private final double m; + private final double b; + + // TODO(kevinb): why so high? why even check this at all? + private static final double EQUALITY_TOLERANCE = 1.0E-6; + + /** + * Constructs a linear translation for which {@code translate(in1) == out1} + * and {@code translate(in2) == out2}. + * + * @throws IllegalArgumentException if {@code in1 == in2} + */ + public LinearTranslation(double in1, double out1, double in2, double out2) { + if (Math.abs(in1 - in2) < EQUALITY_TOLERANCE) { + throw new IllegalArgumentException("in1 and in2 are approximately equal"); + } + double divisor = in1 - in2; + this.m = (out1 - out2) / divisor; + this.b = (in1 * out2 - in2 * out1) / divisor; + } + + public double translate(double in) { + return m * in + b; + } +} diff --git a/caliper/src/main/resources/CaliperCore.gwt.xml b/caliper/src/main/resources/CaliperCore.gwt.xml new file mode 100644 index 0000000..fcfe2fe --- /dev/null +++ b/caliper/src/main/resources/CaliperCore.gwt.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/caliper/src/test/java/com/google/caliper/AllTests.java b/caliper/src/test/java/com/google/caliper/AllTests.java new file mode 100644 index 0000000..0e3d1cc --- /dev/null +++ b/caliper/src/test/java/com/google/caliper/AllTests.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import junit.framework.Test; +import junit.framework.TestSuite; + +public class AllTests { + public static Test suite() { + TestSuite suite = new TestSuite(); + suite.addTestSuite(CaliperTest.class); + suite.addTestSuite(JsonTest.class); + suite.addTestSuite(MeasurementSetTest.class); + suite.addTestSuite(ParameterTest.class); + suite.addTestSuite(WarmupOverflowTest.class); + + return suite; + } +} diff --git a/caliper/src/test/java/com/google/caliper/CaliperTest.java b/caliper/src/test/java/com/google/caliper/CaliperTest.java new file mode 100644 index 0000000..18857ba --- /dev/null +++ b/caliper/src/test/java/com/google/caliper/CaliperTest.java @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.base.Supplier; +import java.util.Map; +import java.util.Set; +import junit.framework.TestCase; + +public final class CaliperTest extends TestCase { + + /** + * Test we detect and fail when benchmarks don't scale properly. + * @throws Exception + */ + public void testBenchmarkScalesNonLinearly() throws Exception { + TimeMeasurer timeMeasurer = new TimeMeasurer(1000, 1000); + try { + timeMeasurer.run(new NonLinearTimedRunnable()); + fail(); + } catch (UserException.DoesNotScaleLinearlyException e) { + } + } + + private static class NonLinearTimedRunnable extends ConfiguredBenchmark + implements Supplier { + private NonLinearTimedRunnable() { + super(new NoOpBenchmark()); + } + + @Override public ConfiguredBenchmark get() { + return this; + } + + @Override public Object run(int reps) throws Exception { + return null; // broken! doesn't loop reps times. + } + + @Override public void close() throws Exception {} + } + + private static class NoOpBenchmark implements Benchmark { + @Override public Set parameterNames() { + return null; + } + + @Override public Set parameterValues(String parameterName) { + return null; + } + + @Override public ConfiguredBenchmark createBenchmark(Map parameterValues) { + return null; + } + + @Override public Map getTimeUnitNames() { + return null; + } + + @Override public Map getInstanceUnitNames() { + return null; + } + + @Override public Map getMemoryUnitNames() { + return null; + } + + @Override public double nanosToUnits(double nanos) { + return 0; + } + + @Override public double instancesToUnits(long instances) { + return 0; + } + + @Override public double bytesToUnits(long bytes) { + return 0; + } + } +} diff --git a/caliper/src/test/java/com/google/caliper/JsonTest.java b/caliper/src/test/java/com/google/caliper/JsonTest.java new file mode 100644 index 0000000..cee02b0 --- /dev/null +++ b/caliper/src/test/java/com/google/caliper/JsonTest.java @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2011 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableMap; +import java.util.Date; +import java.util.Locale; +import java.util.Map; +import junit.framework.TestCase; + +public final class JsonTest extends TestCase { + + public void testJsonSerialization() { + Result original = newSampleResult(); + String json = Json.getGsonInstance().toJson(original, Result.class); + Result reserialized = Json.getGsonInstance().fromJson(json, Result.class); + assertEquals(original, reserialized); + } + + /** + * Caliper's JSON files used to include dates specific to the host machine's + * locale. http://code.google.com/p/caliper/issues/detail?id=113 + */ + public void testJsonSerializationWithFancyLocale() { + Result original = newSampleResult(); + + // serialize in one locale... + Locale defaultLocale = Locale.getDefault(); + Locale.setDefault(Locale.ITALY); + String json; + try { + json = Json.getGsonInstance().toJson(original, Result.class); + } finally { + Locale.setDefault(defaultLocale); + } + + // deserialize in another + Result reserialized = Json.getGsonInstance().fromJson(json, Result.class); + assertEquals(original, reserialized); + } + + private Result newSampleResult() { + Map units = ImmutableMap.of("ns", 1); + MeasurementSet timeMeasurements = new MeasurementSet(new Measurement(units, 2.0, 2.0)); + Date executedDate = new Date(0); + Scenario scenario = new Scenario(ImmutableMap.of("benchmark", "Foo")); + ScenarioResult scenarioResult = new ScenarioResult( + timeMeasurements, "log", null, null, null, null); + Run run = new Run(ImmutableMap.of(scenario, scenarioResult), "foo.FooBenchmark", executedDate); + Environment environment = new Environment(ImmutableMap.of("os.name", "Linux")); + return new Result(run, environment); + } +} diff --git a/caliper/src/test/java/com/google/caliper/MeasurementSetTest.java b/caliper/src/test/java/com/google/caliper/MeasurementSetTest.java new file mode 100644 index 0000000..f6f7cbb --- /dev/null +++ b/caliper/src/test/java/com/google/caliper/MeasurementSetTest.java @@ -0,0 +1,216 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Ordering; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import junit.framework.TestCase; + +public class MeasurementSetTest extends TestCase { + + Ordering MEASUREMENT_BY_NANOS = new Ordering() { + @Override public int compare(Measurement a, Measurement b) { + return Double.compare(a.getRaw(), b.getRaw()); + } + }; + + public void testIncompatibleMeasurements() { + Measurement[] measurements = new Measurement[2]; + measurements[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements[1] = new Measurement(ImmutableMap.of("triplens", 1), 3.8, 7.6); + try { + new MeasurementSet(measurements); + fail("illegal argument exception not thrown"); + } catch (IllegalArgumentException e) { + // success + } + } + + public void testIncompatibleAddedMeasurements() { + Measurement[] measurements = new Measurement[1]; + measurements[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + MeasurementSet measurementSet = new MeasurementSet(measurements); + try { + measurementSet.plusMeasurement(new Measurement(ImmutableMap.of("triplens", 1), 3.8, 7.6)); + fail("illegal argument exception not thrown"); + } catch (IllegalArgumentException e) { + // success + } + } + + public void testSize() { + Measurement[] measurements = new Measurement[3]; + measurements[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + MeasurementSet measurementSet = new MeasurementSet(measurements); + assertEquals(3, measurementSet.size()); + + Measurement[] measurements2 = new Measurement[4]; + measurements2[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements2[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements2[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + measurements2[3] = new Measurement(ImmutableMap.of("doublens", 1), 7.2, 14.4); + MeasurementSet measurementSet2 = + new MeasurementSet(measurements2); + assertEquals(4, measurementSet2.size()); + } + + public void testPlusMeasurement() { + Measurement[] measurements = new Measurement[3]; + measurements[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + MeasurementSet measurementSet = new MeasurementSet(measurements); + + Measurement[] measurements2 = new Measurement[4]; + measurements2[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements2[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements2[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + measurements2[3] = new Measurement(ImmutableMap.of("doublens", 1), 7.2, 14.4); + MeasurementSet measurementSet2 = + new MeasurementSet(measurements2); + + MeasurementSet measurementSet3 = measurementSet.plusMeasurement(measurements2[3]); + + assertDoubleListsEquals(measurementSet2.getMeasurementsRaw(), + measurementSet3.getMeasurementsRaw(), 0.0000001); + assertDoubleListsEquals(measurementSet2.getMeasurementUnits(), + measurementSet3.getMeasurementUnits(), 0.0000001); + assertEquals(measurementSet2.getUnitNames(), measurementSet3.getUnitNames()); + + List measurementList1 = + MEASUREMENT_BY_NANOS.sortedCopy(measurementSet2.getMeasurements()); + List measurementList2 = + MEASUREMENT_BY_NANOS.sortedCopy(measurementSet3.getMeasurements()); + assertEquals(measurementList1.size(), measurementList2.size()); + for (int i = 0; i < measurementList1.size(); i++) { + assertEquals(measurementList1.get(i).getRaw(), + measurementList2.get(i).getRaw()); + assertEquals(measurementList1.get(i).getProcessed(), + measurementList2.get(i).getProcessed()); + } + } + + public void testMedian() { + Measurement[] measurements = new Measurement[3]; + measurements[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + MeasurementSet measurementSet = new MeasurementSet(measurements); + assertEquals(2.3, measurementSet.medianRaw(), 0.00000001); + assertEquals(4.6, measurementSet.medianUnits(), 0.00000001); + + Measurement[] measurements2 = new Measurement[4]; + measurements2[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements2[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements2[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + measurements2[3] = new Measurement(ImmutableMap.of("doublens", 1), 7.2, 14.4); + MeasurementSet measurementSet2 = + new MeasurementSet(measurements2); + assertEquals((2.3 + 3.8) / 2, measurementSet2.medianRaw(), 0.00000001); + assertEquals((4.6 + 7.6) / 2, measurementSet2.medianUnits(), 0.00000001); + } + + public void testMean() { + Measurement[] measurements = new Measurement[3]; + measurements[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + MeasurementSet measurementSet = new MeasurementSet(measurements); + assertEquals((1.1 + 3.8 + 2.3) / 3, measurementSet.meanRaw(), 0.00000001); + assertEquals((2.2 + 7.6 + 4.6) / 3, measurementSet.meanUnits(), 0.00000001); + + Measurement[] measurements2 = new Measurement[4]; + measurements2[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements2[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements2[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + measurements2[3] = new Measurement(ImmutableMap.of("doublens", 1), 7.2, 14.4); + MeasurementSet measurementSet2 = + new MeasurementSet(measurements2); + assertEquals((1.1 + 2.3 + 3.8 + 7.2) / 4, measurementSet2.meanRaw(), 0.00000001); + assertEquals((2.2 + 4.6 + 7.6 + 14.4) / 4, measurementSet2.meanUnits(), 0.00000001); + } + + public void testStandardDeviation() { + Measurement[] measurements = new Measurement[3]; + measurements[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + MeasurementSet measurementSet = new MeasurementSet(measurements); + assertEquals(1.35277, measurementSet.standardDeviationRaw(), 0.00001); + assertEquals(2.70555, measurementSet.standardDeviationUnits(), 0.00001); + } + + public void testMax() { + Measurement[] measurements = new Measurement[3]; + measurements[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + MeasurementSet measurementSet = new MeasurementSet(measurements); + assertEquals(3.8, measurementSet.maxRaw(), 0.00000001); + assertEquals(7.6, measurementSet.maxUnits(), 0.00000001); + } + + public void testMin() { + Measurement[] measurements = new Measurement[3]; + measurements[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + MeasurementSet measurementSet = new MeasurementSet(measurements); + assertEquals(1.1, measurementSet.minRaw(), 0.00000001); + assertEquals(2.2, measurementSet.minUnits(), 0.00000001); + } + + public void testJsonRoundtrip() { + Measurement[] measurements = new Measurement[3]; + measurements[0] = new Measurement(ImmutableMap.of("doublens", 1), 1.1, 2.2); + measurements[1] = new Measurement(ImmutableMap.of("doublens", 1), 3.8, 7.6); + measurements[2] = new Measurement(ImmutableMap.of("doublens", 1), 2.3, 4.6); + MeasurementSet measurementSet = new MeasurementSet(measurements); + MeasurementSet roundTripped = + Json.measurementSetFromJson(Json.measurementSetToJson(measurementSet)); + assertDoubleListsEquals(measurementSet.getMeasurementsRaw(), + roundTripped.getMeasurementsRaw(), 0.00000001); + assertDoubleListsEquals(measurementSet.getMeasurementUnits(), + roundTripped.getMeasurementUnits(), 0.00000001); + assertEquals(measurementSet.getUnitNames(), roundTripped.getUnitNames()); + } + + @SuppressWarnings({"AssertEqualsBetweenInconvertibleTypes"}) + public void testFromLegacyString() { + MeasurementSet measurementSet = Json.measurementSetFromJson("122.0 133.0 144.0"); + assertDoubleListsEquals(Arrays.asList(122.0, 133.0, 144.0), + measurementSet.getMeasurementsRaw(), 0.00000001); + assertDoubleListsEquals(Arrays.asList(122.0, 133.0, 144.0), + measurementSet.getMeasurementUnits(), 0.00000001); + assertEquals(ImmutableMap.of("ns", 1, "us", 1000, "ms", 1000000, "s", 1000000000), + measurementSet.getUnitNames()); + } + + private void assertDoubleListsEquals(List expected, List actual, double epsilon) { + assertEquals(expected.size(), actual.size()); + Collections.sort(expected); + Collections.sort(actual); + for (int i = 0; i < expected.size(); i++) { + assertEquals(expected.get(i), actual.get(i), epsilon); + } + } +} diff --git a/caliper/src/test/java/com/google/caliper/ParameterTest.java b/caliper/src/test/java/com/google/caliper/ParameterTest.java new file mode 100644 index 0000000..9063261 --- /dev/null +++ b/caliper/src/test/java/com/google/caliper/ParameterTest.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; + +import junit.framework.TestCase; + +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; + +public class ParameterTest extends TestCase { + + public static class A extends SimpleBenchmark { + @Param({"value1", "value2"}) String param; + } + + public void testFromAnnotation() throws Exception { + Map> map = Parameter.forClass(A.class); + Parameter p = map.get("param"); + assertEquals("param", p.getName()); + assertEquals(String.class, p.getType()); + + checkParameterValues(A.class, "value1", "value2"); + } + + public enum Foo { VALUE1, VALUE2 } + + public static class H extends SimpleBenchmark { + @Param Foo param; + } + + public void testAllEnums() throws Exception { + checkParameterValues(H.class, Foo.VALUE1, Foo.VALUE2); + } + + public static class I extends SimpleBenchmark { + @Param boolean param; + } + + public void testBoolean() throws Exception { + checkParameterValues(I.class, true, false); + } + + private static void checkParameterValues(Class bClass, + Object... expected) throws Exception { + Map> map = Parameter.forClass(bClass); + assertEquals(1, map.size()); + Parameter p = map.get("param"); + List values = ImmutableList.copyOf(p.values()); + assertEquals(Arrays.asList(expected), values); + } +} diff --git a/caliper/src/test/java/com/google/caliper/WarmupOverflowTest.java b/caliper/src/test/java/com/google/caliper/WarmupOverflowTest.java new file mode 100644 index 0000000..85e59af --- /dev/null +++ b/caliper/src/test/java/com/google/caliper/WarmupOverflowTest.java @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.caliper; + +import com.google.caliper.UserException.DoesNotScaleLinearlyException; +import com.google.common.util.concurrent.SimpleTimeLimiter; +import com.google.common.util.concurrent.TimeLimiter; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import junit.framework.TestCase; + +/** + * Test exposing an issue where any warmup that completes enough executions to reach + * Integer.MAX_VALUE reps (either because the benchmark code is optimized away or because the + * warmupMillis are long enough compared to the benchmark execution time). + */ +public class WarmupOverflowTest extends TestCase { + private TimeLimiter timeLimiter; + + @Override public void setUp() { + timeLimiter = new SimpleTimeLimiter(Executors.newSingleThreadExecutor()); + } + + public void testOptimizedAwayBenchmarkDoesNotTakeTooLongToRun() throws Exception { + try { + timeLimiter.callWithTimeout(new Callable() { + @Override public Void call() throws Exception { + InProcessRunner runner = new InProcessRunner(); + runner.run(OptimizedAwayBenchmark.class.getName(), "--warmupMillis", "3000", + "--measurementType", "TIME"); + return null; + } + }, 90, TimeUnit.SECONDS, false); + } catch (DoesNotScaleLinearlyException expected) { + } + } + + public void testLongWarmupMillisDoesNotTakeTooLongToRun() throws Exception { + timeLimiter.callWithTimeout(new Callable() { + @Override public Void call() throws Exception { + InProcessRunner runner = new InProcessRunner(); + runner.run(RelativelyFastBenchmark.class.getName(), "--warmupMillis", "8000", + "--runMillis", "51", "--measurementType", "TIME"); + return null; + } + }, 90, TimeUnit.SECONDS, false); + } + + public static class OptimizedAwayBenchmark extends SimpleBenchmark { + public void timeIsNullOrEmpty(int reps) { + for (int i = 0; i < reps; i++) { + // do nothing! + } + } + } + + public static class RelativelyFastBenchmark extends SimpleBenchmark { + public long timeSqrt(int reps) { + long result = 0; + for(int i = 0; i < reps; i++) { + result += Math.sqrt(81); + } + return result; + } + } +} diff --git a/examples/pom.xml b/examples/pom.xml new file mode 100644 index 0000000..7352d7b --- /dev/null +++ b/examples/pom.xml @@ -0,0 +1,88 @@ + + 4.0.0 + com.google.caliper + caliper-examples + jar + 0.5-rc1 + 2009 + Caliper Examples + + org.sonatype.oss + oss-parent + 5 + + http://code.google.com/p/caliper/ + Caliper Examples + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + scm:git:http://code.google.com/p/caliper/examples + scm:git:git:http://code.google.com/p/caliper/examples + http://caliper.codegoogle.com/svn/trunk/examples + + + Google Code Issue Tracking + http://code.google.com/p/caliper/issues/list + + + Google, Inc. + http://www.google.com + + + + com.google.caliper + caliper + 0.5-rc1 + + + junit + junit + 3.8.2 + test + + + + package + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 1.6 + 1.6 + + + + org.apache.maven.plugins + maven-eclipse-plugin + 2.8 + + true + true + ../eclipse-ws/ + + + + org.apache.maven.plugins + maven-release-plugin + 2.1 + + -DenableCiProfile=true + + + + + + + Jesse Wilson + Google Inc. + + + diff --git a/examples/src/main/java/examples/ArraySortBenchmark.java b/examples/src/main/java/examples/ArraySortBenchmark.java new file mode 100644 index 0000000..f42390f --- /dev/null +++ b/examples/src/main/java/examples/ArraySortBenchmark.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; +import java.util.Arrays; +import java.util.Random; + +/** + * Measures sorting on different distributions of integers. + */ +public class ArraySortBenchmark extends SimpleBenchmark { + + @Param({"10", "100", "1000", "10000"}) private int length; + + @Param private Distribution distribution; + + private int[] values; + private int[] copy; + + @Override protected void setUp() throws Exception { + values = distribution.create(length); + copy = new int[length]; + } + + public void timeSort(int reps) { + for (int i = 0; i < reps; i++) { + System.arraycopy(values, 0, copy, 0, values.length); + Arrays.sort(copy); + } + } + + public enum Distribution { + SAWTOOTH { + @Override + int[] create(int length) { + int[] result = new int[length]; + for (int i = 0; i < length; i += 5) { + result[i] = 0; + result[i + 1] = 1; + result[i + 2] = 2; + result[i + 3] = 3; + result[i + 4] = 4; + } + return result; + } + }, + INCREASING { + @Override + int[] create(int length) { + int[] result = new int[length]; + for (int i = 0; i < length; i++) { + result[i] = i; + } + return result; + } + }, + DECREASING { + @Override + int[] create(int length) { + int[] result = new int[length]; + for (int i = 0; i < length; i++) { + result[i] = length - i; + } + return result; + } + }, + RANDOM { + @Override + int[] create(int length) { + Random random = new Random(); + int[] result = new int[length]; + for (int i = 0; i < length; i++) { + result[i] = random.nextInt(); + } + return result; + } + }; + + abstract int[] create(int length); + } + + public static void main(String[] args) throws Exception { + Runner.main(ArraySortBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/BitSetBenchmark.java b/examples/src/main/java/examples/BitSetBenchmark.java new file mode 100644 index 0000000..3154b01 --- /dev/null +++ b/examples/src/main/java/examples/BitSetBenchmark.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; +import java.util.BitSet; +import java.util.Random; + +/** + * A simple example of a benchmark for BitSet showing some of the issues with + * micro-benchmarking. + * + *

The following is a discussion of how the benchmarks evolved and what they + * may (or may not) tell us. This discussion is based on the following set of + * results: + * + * 

+ *  0% Scenario{vm=java, benchmark=SetBitSetX64} 233.45ns; σ=0.31ns @ 3 trials
+ * 20% Scenario{vm=java, benchmark=SetMaskX64} 116.62ns; σ=0.09ns @ 3 trials
+ * 40% Scenario{vm=java, benchmark=CharsToBitSet} 748.40ns; σ=23.52ns @ 10 trials
+ * 60% Scenario{vm=java, benchmark=CharsToMask} 198.55ns; σ=9.46ns @ 10 trials
+ * 80% Scenario{vm=java, benchmark=BaselineIteration} 67.85ns; σ=0.44ns @ 3 trials
+ *
+ *         benchmark   ns logarithmic runtime
+ *      SetBitSetX64  233 XXXXXXXXX|||||||||||||||
+ *        SetMaskX64  117 XXXX|||||||||||||||||
+ *     CharsToBitSet  748 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+ *       CharsToMask  199 XXXXXXX||||||||||||||||
+ * BaselineIteration   68 XX|||||||||||||||||
+ *
+ * + *

Initially things look simple. The {@link #timeSetBitSetX64(int)} benchmark + * takes approximately twice as long as {@link #timeSetMaskX64(int)}. However + * the inner loops in these benchmarks have almost no content, so a more + * 'real world' benchmark was devised in an attempt to back up these results. + * + *

The {@link #timeCharsToMask(int)} and {@link #timeCharsToBitSet(int)} + * benchmarks convert a simple char[] of '1's and '0's to a corresponding BitSet + * or bit mask. These also processes 64 bits per iteration and so appears to be + * doing the same amount of work as the first benchmarks. + * + *

Additionally the {@link BitSetBenchmark#timeBaselineIteration(int)} + * benchmark attempts to measure the raw cost of looping through and reading the + * source data. + * + *

When comparing the benchmarks that use bit masking, we see that the + * measured time of the SetMaskX64 benchmark (117ns) is roughly the same + * as the CharsToMask benchmark (199ns) with the BaselineIteration time (68ms) + * subtracted from it. This gives us some confidence that both benchmarks are + * resulting in the same underlying work on the CPU. + * + *

However the CharsToBitSet and the SetBitSetX64 benchmarks differ very + * significantly (approximately 3x) even when accounting for the + * BaselineIteration result. This suggests that the performance of + * {@link BitSet#set} is quite dependent on the surrounding code and how + * it is optimized by the JVM. + * + *

The conclusions we can draw from this are: + * + *

1: Using BitSet is slower than using bit masks directly. At best it + * seems about 2x slower than a bit mask, but could easily be 5x slower in real + * applications. + * + *

While these are only estimates, we can conclude that when performance is + * important and where bit set operations occur in tight loops, bit masks + * should be used in favor of BitSets. + * + *

2:Overly simplistic benchmarks can give a very false impression of + * performance. + */ +public class BitSetBenchmark extends SimpleBenchmark { + private BitSet bitSet; + private char[] bitString; + + @Override protected void setUp() throws Exception { + bitSet = new BitSet(64); + bitString = new char[64]; + Random r = new Random(); + for (int n = 0; n < 64; n++) { + bitString[n] = r.nextBoolean() ? '1' : '0'; + } + } + + /** + * This benchmark attempts to measure performance of {@link BitSet#set}. + */ + public int timeSetBitSetX64(int reps) { + long count = 64L * reps; + for (int i = 0; i < count; i++) { + bitSet.set(i & 0x3F, true); + } + return bitSet.hashCode(); + } + + /** + * This benchmark attempts to measure performance of direct bit-manipulation. + */ + public long timeSetMaskX64(int reps) { + long count = 64L * reps; + long bitMask = 0L; + for (int i = 0; i < count; i++) { + bitMask |= 1 << (i & 0x3F); + } + return bitMask; + } + + /** + * This benchmark parses a char[] of 1's and 0's into a BitSet. Results from + * this benchmark should be comparable with those from + * {@link #timeCharsToMask(int)}. + */ + public String timeCharsToBitSet(int reps) { + /* + * This benchmark now measures the complete parsing of a char[] rather than + * a single invocation of {@link BitSet#set}. However this fine because + * it is intended to be a comparative benchmark. + */ + for (int i = 0; i < reps; i++) { + for (int n = 0; n < bitString.length; n++) { + bitSet.set(n, bitString[n] == '1'); + } + } + return bitSet.toString(); + } + + /** + * This benchmark parses a char[] of 1's and 0's into a bit mask. Results from + * this benchmark should be comparable with those from + * {@link #timeCharsToBitSet(int)}. + */ + public long timeCharsToMask(int reps) { + /* + * Comparing results we see a far more realistic sounding result whereby + * using a bit mask is a little over 4x faster than using BitSet. + */ + long bitMask = 0; + for (int i = 0; i < reps; i++) { + for (int n = 0; n < bitString.length; n++) { + long m = 1 << n; + if (bitString[n] == '1') { + bitMask |= m; + } else { + bitMask &= ~m; + } + } + } + return bitMask; + } + + /** + * This benchmark attempts to measure the baseline cost of both + * {@link #timeCharsToBitSet(int)} and {@link #timeCharsToMask(int)}. + * It does this by unconditionally summing the character values of the char[]. + * This is as close to a no-op case as we can expect to get without unwanted + * over-optimization. + */ + public long timeBaselineIteration(int reps) { + int badHash = 0; + for (int i = 0; i < reps; i++) { + for (int n = 0; n < bitString.length; n++) { + badHash += bitString[n]; + } + } + return badHash; + } + + // TODO: remove this from all examples when IDE plugins are ready + public static void main(String[] args) throws Exception { + Runner.main(BitSetBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/CharacterBenchmark.java b/examples/src/main/java/examples/CharacterBenchmark.java new file mode 100644 index 0000000..1e013af --- /dev/null +++ b/examples/src/main/java/examples/CharacterBenchmark.java @@ -0,0 +1,300 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; + +/** + * Tests various Character methods, intended for testing multiple + * implementations against each other. + */ +public class CharacterBenchmark extends SimpleBenchmark { + + @Param private CharacterSet characterSet; + + @Param private Overload overload; + + private char[] chars; + + @Override protected void setUp() throws Exception { + this.chars = characterSet.chars; + } + + public enum Overload { CHAR, INT } + + public enum CharacterSet { + ASCII(128), + UNICODE(65536); + final char[] chars; + CharacterSet(int size) { + this.chars = new char[65536]; + for (int i = 0; i < 65536; ++i) { + chars[i] = (char) (i % size); + } + } + } + + // A fake benchmark to give us a baseline. + public boolean timeIsSpace(int reps) { + boolean dummy = false; + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + dummy ^= ((char) ch == ' '); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + dummy ^= (ch == ' '); + } + } + } + return dummy; + } + + public void timeDigit(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.digit(chars[ch], 10); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.digit((int) chars[ch], 10); + } + } + } + } + + public void timeGetNumericValue(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.getNumericValue(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.getNumericValue((int) chars[ch]); + } + } + } + } + + public void timeIsDigit(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isDigit(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isDigit((int) chars[ch]); + } + } + } + } + + public void timeIsIdentifierIgnorable(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isIdentifierIgnorable(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isIdentifierIgnorable((int) chars[ch]); + } + } + } + } + + public void timeIsJavaIdentifierPart(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isJavaIdentifierPart(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isJavaIdentifierPart((int) chars[ch]); + } + } + } + } + + public void timeIsJavaIdentifierStart(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isJavaIdentifierStart(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isJavaIdentifierStart((int) chars[ch]); + } + } + } + } + + public void timeIsLetter(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isLetter(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isLetter((int) chars[ch]); + } + } + } + } + + public void timeIsLetterOrDigit(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isLetterOrDigit(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isLetterOrDigit((int) chars[ch]); + } + } + } + } + + public void timeIsLowerCase(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isLowerCase(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isLowerCase((int) chars[ch]); + } + } + } + } + + public void timeIsSpaceChar(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isSpaceChar(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isSpaceChar((int) chars[ch]); + } + } + } + } + + public void timeIsUpperCase(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isUpperCase(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isUpperCase((int) chars[ch]); + } + } + } + } + + public void timeIsWhitespace(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isWhitespace(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.isWhitespace((int) chars[ch]); + } + } + } + } + + public void timeToLowerCase(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.toLowerCase(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.toLowerCase((int) chars[ch]); + } + } + } + } + + public void timeToUpperCase(int reps) { + if (overload == Overload.CHAR) { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.toUpperCase(chars[ch]); + } + } + } else { + for (int i = 0; i < reps; ++i) { + for (int ch = 0; ch < 65536; ++ch) { + Character.toUpperCase((int) chars[ch]); + } + } + } + } + + // TODO: remove this from all examples when IDE plugins are ready + public static void main(String[] args) throws Exception { + Runner.main(CharacterBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/CompressionSizeBenchmark.java b/examples/src/main/java/examples/CompressionSizeBenchmark.java new file mode 100644 index 0000000..90ddd39 --- /dev/null +++ b/examples/src/main/java/examples/CompressionSizeBenchmark.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.api.Benchmark; +import com.google.caliper.model.ArbitraryMeasurement; +import com.google.caliper.runner.CaliperMain; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.zip.Deflater; + +/** + * Example "arbitrary measurement" benchmark. + */ +public class CompressionSizeBenchmark extends Benchmark { + + @Param({ + "this string will compress badly", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf"}) + private String toCompress; + @Param({"bestCompression", "bestSpeed", "noCompression", "huffmanOnly"}) + private String compressionLevel; + + private double compressionRatio; + + public static final Map compressionLevelMap = new HashMap(); + static { + compressionLevelMap.put("bestCompression", Deflater.BEST_COMPRESSION); + compressionLevelMap.put("bestSpeed", Deflater.BEST_SPEED); + compressionLevelMap.put("noCompression", Deflater.NO_COMPRESSION); + compressionLevelMap.put("huffmanOnly", Deflater.HUFFMAN_ONLY); + } + + public long timeSimpleCompression(int reps) { + long dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += compress(toCompress.getBytes()).length; + } + return dummy; + } + + @ArbitraryMeasurement(units = ":1", description = "ratio of uncompressed to compressed") + public double compressionSize() { + byte[] initialBytes = toCompress.getBytes(); + byte[] finalBytes = compress(initialBytes); + compressionRatio = (double) initialBytes.length / (double) finalBytes.length; + return compressionRatio; + } + + private byte[] compress(byte[] bytes) { + Deflater compressor = new Deflater(); + compressor.setLevel(compressionLevelMap.get(compressionLevel)); + compressor.setInput(bytes); + compressor.finish(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + byte[] buf = new byte[1024]; + while (!compressor.finished()) { + int count = compressor.deflate(buf); + bos.write(buf, 0, count); + } + try { + bos.close(); + } catch (IOException e) { + } + return bos.toByteArray(); + } + + public static void main(String[] args) { + CaliperMain.main(CompressionSizeBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/ContainsBenchmark.java b/examples/src/main/java/examples/ContainsBenchmark.java new file mode 100644 index 0000000..01bb8c6 --- /dev/null +++ b/examples/src/main/java/examples/ContainsBenchmark.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +public class ContainsBenchmark extends SimpleBenchmark { + @Param({"0", "25", "50", "75", "100"}) private int percentNulls; + @Param({"100", "1000", "10000"}) private int containsPerRep; + + /** the set under test */ + private final Set set = new HashSet(); + + /** twenty-five percent nulls */ + private final List queries = new ArrayList(); + + @Override protected void setUp() { + set.addAll(Arrays.asList("str1", "str2", "str3", "str4")); + int nullThreshold = percentNulls * containsPerRep / 100; + for (int i = 0; i < nullThreshold; i++) { + queries.add(null); + } + for (int i = nullThreshold; i < containsPerRep; i++) { + queries.add(new Object()); + } + Collections.shuffle(queries, new Random(0)); + } + + @Override public Map getTimeUnitNames() { + Map unitNames = new HashMap(); + unitNames.put("ns/contains", 1); + unitNames.put("us/contains", 1000); + unitNames.put("ms/contains", 1000000); + unitNames.put("s/contains", 1000000000); + return unitNames; + } + + @Override public double nanosToUnits(double nanos) { + return nanos / containsPerRep; + } + + public void timeContains(int reps) { + for (int i = 0; i < reps; i++) { + for (Object query : queries) { + set.contains(query); + } + } + } + + public static void main(String[] args) { + Runner.main(ContainsBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/CopyArrayBenchmark.java b/examples/src/main/java/examples/CopyArrayBenchmark.java new file mode 100644 index 0000000..fed0950 --- /dev/null +++ b/examples/src/main/java/examples/CopyArrayBenchmark.java @@ -0,0 +1,356 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; + +import java.util.Arrays; +import java.util.Random; + +/** + * Tests each of four ways to copy an array, for all nine array types. + * + *

Once upon a time, {@code clone} was much slower than the other array copy techniques, but + * that was fixed in Sun bug: + * + * + * array clone() much slower than Arrays.copyOf + * + * at which time all copy methods were equally efficient. + * + *

Recent (2011) measurements with OpenJDK 7 on Linux are less clear. Results suggests that: + * + *

    + *
  • The different methods of copying have indistinguishable performance with hotspot server for + * all nine types, except that the naive LOOP is slower. + * With the "client" compiler, LOOP beats CLONE, which is the slowest. + *
  • As array sizes get large, the runtime is indeed proportional to the size of the array in + * memory (boolean arrays count as byte arrays!). + *
+ */ +public class CopyArrayBenchmark extends SimpleBenchmark { + public enum Strategy { + CLONE { + @Override Object[] copy(Object[] array) { + return array.clone(); + } + @Override boolean[] copy(boolean[] array) { + return array.clone(); + } + @Override byte[] copy(byte[] array) { + return array.clone(); + } + @Override char[] copy(char[] array) { + return array.clone(); + } + @Override double[] copy(double[] array) { + return array.clone(); + } + @Override float[] copy(float[] array) { + return array.clone(); + } + @Override int[] copy(int[] array) { + return array.clone(); + } + @Override long[] copy(long[] array) { + return array.clone(); + } + @Override short[] copy(short[] array) { + return array.clone(); + } + }, + ARRAYS_COPYOF { + @Override Object[] copy(Object[] array) { + return Arrays.copyOf(array, array.length); + } + @Override boolean[] copy(boolean[] array) { + return Arrays.copyOf(array, array.length); + } + @Override byte[] copy(byte[] array) { + return Arrays.copyOf(array, array.length); + } + @Override char[] copy(char[] array) { + return Arrays.copyOf(array, array.length); + } + @Override double[] copy(double[] array) { + return Arrays.copyOf(array, array.length); + } + @Override float[] copy(float[] array) { + return Arrays.copyOf(array, array.length); + } + @Override int[] copy(int[] array) { + return Arrays.copyOf(array, array.length); + } + @Override long[] copy(long[] array) { + return Arrays.copyOf(array, array.length); + } + @Override short[] copy(short[] array) { + return Arrays.copyOf(array, array.length); + } + }, + SYSTEM_ARRAYCOPY { + @Override Object[] copy(Object[] array) { + Object[] copy = new Object[array.length]; + System.arraycopy(array, 0, copy, 0, array.length); + return copy; + } + @Override boolean[] copy(boolean[] array) { + boolean[] copy = new boolean[array.length]; + System.arraycopy(array, 0, copy, 0, array.length); + return copy; + } + @Override byte[] copy(byte[] array) { + byte[] copy = new byte[array.length]; + System.arraycopy(array, 0, copy, 0, array.length); + return copy; + } + @Override char[] copy(char[] array) { + char[] copy = new char[array.length]; + System.arraycopy(array, 0, copy, 0, array.length); + return copy; + } + @Override double[] copy(double[] array) { + double[] copy = new double[array.length]; + System.arraycopy(array, 0, copy, 0, array.length); + return copy; + } + @Override float[] copy(float[] array) { + float[] copy = new float[array.length]; + System.arraycopy(array, 0, copy, 0, array.length); + return copy; + } + @Override int[] copy(int[] array) { + int[] copy = new int[array.length]; + System.arraycopy(array, 0, copy, 0, array.length); + return copy; + } + @Override long[] copy(long[] array) { + long[] copy = new long[array.length]; + System.arraycopy(array, 0, copy, 0, array.length); + return copy; + } + @Override short[] copy(short[] array) { + short[] copy = new short[array.length]; + System.arraycopy(array, 0, copy, 0, array.length); + return copy; + } + }, + LOOP { + @Override Object[] copy(Object[] array) { + int len = array.length; + Object[] copy = new Object[len]; + for (int i = 0; i < len; i++) { + copy[i] = array[i]; + } + return copy; + } + @Override boolean[] copy(boolean[] array) { + int len = array.length; + boolean[] copy = new boolean[array.length]; + for (int i = 0; i < len; i++) { + copy[i] = array[i]; + } + return copy; + } + @Override byte[] copy(byte[] array) { + int len = array.length; + byte[] copy = new byte[array.length]; + for (int i = 0; i < len; i++) { + copy[i] = array[i]; + } + return copy; + } + @Override char[] copy(char[] array) { + int len = array.length; + char[] copy = new char[array.length]; + for (int i = 0; i < len; i++) { + copy[i] = array[i]; + } + return copy; + } + @Override double[] copy(double[] array) { + int len = array.length; + double[] copy = new double[array.length]; + for (int i = 0; i < len; i++) { + copy[i] = array[i]; + } + return copy; + } + @Override float[] copy(float[] array) { + int len = array.length; + float[] copy = new float[array.length]; + for (int i = 0; i < len; i++) { + copy[i] = array[i]; + } + return copy; + } + @Override int[] copy(int[] array) { + int len = array.length; + int[] copy = new int[array.length]; + for (int i = 0; i < len; i++) { + copy[i] = array[i]; + } + return copy; + } + @Override long[] copy(long[] array) { + int len = array.length; + long[] copy = new long[array.length]; + for (int i = 0; i < len; i++) { + copy[i] = array[i]; + } + return copy; + } + @Override short[] copy(short[] array) { + int len = array.length; + short[] copy = new short[array.length]; + for (int i = 0; i < len; i++) { + copy[i] = array[i]; + } + return copy; + } + }, + ; + + abstract Object[] copy(Object[] array); + abstract boolean[] copy(boolean[] array); + abstract byte[] copy(byte[] array); + abstract char[] copy(char[] array); + abstract double[] copy(double[] array); + abstract float[] copy(float[] array); + abstract int[] copy(int[] array); + abstract long[] copy(long[] array); + abstract short[] copy(short[] array); + } + + @Param Strategy strategy; + + @Param({"5", "500", "50000"}) int size; + + Object[] objectArray; + boolean[] booleanArray; + byte[] byteArray; + char[] charArray; + double[] doubleArray; + float[] floatArray; + int[] intArray; + long[] longArray; + short[] shortArray; + + @Override protected void setUp() { + objectArray = new Object[size]; + booleanArray = new boolean[size]; + byteArray = new byte[size]; + charArray = new char[size]; + doubleArray = new double[size]; + floatArray = new float[size]; + intArray = new int[size]; + longArray = new long[size]; + shortArray = new short[size]; + + Random random = new Random(); + for (int i = 0; i < size; i++) { + int num = random.nextInt(); + objectArray[i] = new Object(); + booleanArray[i] = num % 2 == 0; + byteArray[i] = (byte) num; + charArray[i] = (char) num; + doubleArray[i] = num; + floatArray[i] = (float) num; + intArray[i] = num; + longArray[i] = num; + shortArray[i] = (short) num; + } + } + + public int timeObjects(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += strategy.copy(objectArray).hashCode(); + } + return dummy; + } + + public int timeBooleans(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += strategy.copy(booleanArray).hashCode(); + } + return dummy; + } + + public int timeBytes(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += strategy.copy(byteArray).hashCode(); + } + return dummy; + } + + public int timeChars(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += strategy.copy(charArray).hashCode(); + } + return dummy; + } + + public int timeDoubles(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += strategy.copy(doubleArray).hashCode(); + } + return dummy; + } + + public int timeFloats(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += strategy.copy(floatArray).hashCode(); + } + return dummy; + } + + public int timeInts(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += strategy.copy(intArray).hashCode(); + } + return dummy; + } + + public int timeLongs(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += strategy.copy(longArray).hashCode(); + } + return dummy; + } + + public int timeShorts(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += strategy.copy(shortArray).hashCode(); + } + return dummy; + } + + public static void main(String[] args) { + Runner.main(CopyArrayBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/DemoBenchmark.java b/examples/src/main/java/examples/DemoBenchmark.java new file mode 100644 index 0000000..3b7e2dd --- /dev/null +++ b/examples/src/main/java/examples/DemoBenchmark.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2011 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.api.Benchmark; +import com.google.caliper.api.SkipThisScenarioException; +import com.google.caliper.api.VmParam; +import com.google.caliper.runner.CaliperMain; +import com.google.caliper.util.ShortDuration; + +import java.math.BigDecimal; + +public class DemoBenchmark extends Benchmark { + @Param({"abc", "def", "xyz"}) String string; + @Param({"1", "2"}) int number; + @Param Foo foo; + + @Param({"0.00", "123.45"}) BigDecimal money; + @Param({"1ns", "2 minutes"}) ShortDuration duration; + @VmParam({"-Xmx32m", "-Xmx1g"}) String memoryMax; + + enum Foo { + FOO, BAR, BAZ, QUX; + } + + DemoBenchmark() { +// System.out.println("I should not do this."); + } + + @Override protected void setUp() throws Exception { + if (string.equals("abc") && number == 1) { + throw new SkipThisScenarioException(); + } + } + + public int timeSomething(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += i; + } + return dummy; + } + + public int timeSomethingElse(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy -= i; + } + return dummy; + } + + @Override protected void tearDown() throws Exception { +// System.out.println("Hey, I'm tearing up the joint."); + } + + public static void main(String[] args) { + CaliperMain.main(DemoBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/DoubleToStringBenchmark.java b/examples/src/main/java/examples/DoubleToStringBenchmark.java new file mode 100644 index 0000000..291ec84 --- /dev/null +++ b/examples/src/main/java/examples/DoubleToStringBenchmark.java @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; + +import java.util.Arrays; +import java.util.List; + +/** + * Measures the various ways the JDK converts doubles to strings. + */ +public class DoubleToStringBenchmark extends SimpleBenchmark { + @Param Method method; + + public enum Method { + TO_STRING { + @Override String convert(double d) { + return ((Double) d).toString(); + } + @Override String convert(Double d) { + return d.toString(); + } + }, + STRING_VALUE_OF { + @Override String convert(double d) { + return String.valueOf(d); + } + @Override String convert(Double d) { + return String.valueOf(d); + } + }, + STRING_FORMAT { + @Override String convert(double d) { + return String.format("%f", d); + } + @Override String convert(Double d) { + return String.format("%f", d); + } + }, + QUOTE_TRICK { + @Override String convert(double d) { + return "" + d; + } + @Override String convert(Double d) { + return "" + d; + } + }, + ; + + abstract String convert(double d); + abstract String convert(Double d); + } + + enum Value { + Pi(Math.PI), + NegativeZero(-0.0), + NegativeInfinity(Double.NEGATIVE_INFINITY), + NaN(Double.NaN); + + final double value; + + Value(double value) { + this.value = value; + } + } + + @Param Value value; + + public int timePrimitive(int reps) { + double d = value.value; + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += method.convert(d).length(); + } + return dummy; + } + + public int timeWrapper(int reps) { + Double d = value.value; + int dummy = 0; + for (int i = 0; i < reps; i++) { + dummy += method.convert(d).length(); + } + return dummy; + } + + public static void main(String[] args) throws Exception { + Runner.main(DoubleToStringBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/DoubleToStringBenchmark2.java b/examples/src/main/java/examples/DoubleToStringBenchmark2.java new file mode 100644 index 0000000..8e6c69a --- /dev/null +++ b/examples/src/main/java/examples/DoubleToStringBenchmark2.java @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; + +/** + * Measures the various ways the JDK converts doubles to strings. + */ +public class DoubleToStringBenchmark2 extends SimpleBenchmark { + @Param boolean useWrapper; + + enum Value { + Pi(Math.PI), + NegativeZero(-0.0), + NegativeInfinity(Double.NEGATIVE_INFINITY), + NaN(Double.NaN); + + final double d; + + Value(double d) { + this.d = d; + } + } + + @Param Value value; + + public int timeToString(int reps) { + int dummy = 0; + if (useWrapper) { + Double d = value.d; + for (int i = 0; i < reps; i++) { + dummy += d.toString().length(); + } + } else { + double d = value.d; + for (int i = 0; i < reps; i++) { + dummy += ((Double) d).toString().length(); + } + } + return dummy; + } + + public int timeStringValueOf(int reps) { + int dummy = 0; + if (useWrapper) { + Double d = value.d; + for (int i = 0; i < reps; i++) { + dummy += String.valueOf(d).length(); + } + } else { + double d = value.d; + for (int i = 0; i < reps; i++) { + dummy += String.valueOf(d).length(); + } + } + return dummy; + } + + public int timeStringFormat(int reps) { + int dummy = 0; + if (useWrapper) { + Double d = value.d; + for (int i = 0; i < reps; i++) { + dummy += String.format("%f", d).length(); + } + } else { + double d = value.d; + for (int i = 0; i < reps; i++) { + dummy += String.format("%f", d).length(); + } + } + return dummy; + } + + public int timeQuoteTrick(int reps) { + int dummy = 0; + if (useWrapper) { + Double d = value.d; + for (int i = 0; i < reps; i++) { + dummy += ("" + d).length(); + } + } else { + double d = value.d; + for (int i = 0; i < reps; i++) { + dummy += ("" + d).length(); + } + } + return dummy; + } + + public static void main(String[] args) throws Exception { + Runner.main(DoubleToStringBenchmark2.class, args); + } +} diff --git a/examples/src/main/java/examples/EnumSetContainsBenchmark.java b/examples/src/main/java/examples/EnumSetContainsBenchmark.java new file mode 100644 index 0000000..b232514 --- /dev/null +++ b/examples/src/main/java/examples/EnumSetContainsBenchmark.java @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; +import java.util.EnumSet; +import java.util.Set; + +/** + * Measures EnumSet#contains(). + */ +public class EnumSetContainsBenchmark extends SimpleBenchmark { + + @Param private SetMaker setMaker; + + public enum SetMaker { + ENUM_SET { + @Override Set newSet() { + return EnumSet.allOf(RegularSize.class); + } + @Override Object[] testValues() { + return new Object[] { RegularSize.E1, RegularSize.E2, RegularSize.E20, + RegularSize.E39, RegularSize.E40, "A", LargeSize.E40, null }; + } + }, + LARGE_ENUM_SET { + @Override Set newSet() { + return EnumSet.allOf(LargeSize.class); + } + @Override Object[] testValues() { + return new Object[] { LargeSize.E1, LargeSize.E63, LargeSize.E64, + LargeSize.E65, LargeSize.E140, "A", RegularSize.E40, null }; + } + }; + + abstract Set newSet(); + abstract Object[] testValues(); + } + + private enum RegularSize { + E1, E2, E3, E4, E5, E6, E7, E8, E9, E10, E11, E12, E13, E14, E15, E16, E17, + E18, E19, E20, E21, E22, E23, E24, E25, E26, E27, E28, E29, E30, E31, E32, + E33, E34, E35, E36, E37, E38, E39, E40, + } + + private enum LargeSize { + E1, E2, E3, E4, E5, E6, E7, E8, E9, E10, E11, E12, E13, E14, E15, E16, E17, + E18, E19, E20, E21, E22, E23, E24, E25, E26, E27, E28, E29, E30, E31, E32, + E33, E34, E35, E36, E37, E38, E39, E40, E41, E42, E43, E44, E45, E46, E47, + E48, E49, E50, E51, E52, E53, E54, E55, E56, E57, E58, E59, E60, E61, E62, + E63, E64, E65, E66, E67, E68, E69, E70, E71, E72, E73, E74, E75, E76, E77, + E78, E79, E80, E81, E82, E83, E84, E85, E86, E87, E88, E89, E90, E91, E92, + E93, E94, E95, E96, E97, E98, E99, E100, E101, E102, E103, E104, E105, E106, + E107, E108, E109, E110, E111, E112, E113, E114, E115, E116, E117, E118, + E119, E120, E121, E122, E123, E124, E125, E126, E127, E128, E129, E130, + E131, E132, E133, E134, E135, E136, E137, E138, E139, E140, + } + + private Set set; + private Object[] testValues; + + @Override protected void setUp() { + this.set = setMaker.newSet(); + this.testValues = setMaker.testValues(); + } + + public void timeContains(int reps) { + for (int i = 0; i < reps; i++) { + set.contains(testValues[i % testValues.length]); + } + } + + public static void main(String[] args) throws Exception { + Runner.main(EnumSetContainsBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/ExpensiveObjectsBenchmark.java b/examples/src/main/java/examples/ExpensiveObjectsBenchmark.java new file mode 100644 index 0000000..a11b1bd --- /dev/null +++ b/examples/src/main/java/examples/ExpensiveObjectsBenchmark.java @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.SimpleBenchmark; +import com.google.caliper.runner.CaliperMain; + +import java.text.DecimalFormatSymbols; +import java.text.NumberFormat; +import java.text.SimpleDateFormat; +import java.util.Locale; + +/** + * Benchmarks creation and cloning various expensive objects. + */ +@SuppressWarnings({"ResultOfObjectAllocationIgnored"}) // TODO: should fix! +public class ExpensiveObjectsBenchmark extends SimpleBenchmark { + public void timeNewDecimalFormatSymbols(int reps) { + for (int i = 0; i < reps; ++i) { + new DecimalFormatSymbols(Locale.US); + } + } + + public void timeClonedDecimalFormatSymbols(int reps) { + DecimalFormatSymbols dfs = new DecimalFormatSymbols(Locale.US); + for (int i = 0; i < reps; ++i) { + dfs.clone(); + } + } + + public void timeNewNumberFormat(int reps) { + for (int i = 0; i < reps; ++i) { + NumberFormat.getInstance(Locale.US); + } + } + + public void timeClonedNumberFormat(int reps) { + NumberFormat nf = NumberFormat.getInstance(Locale.US); + for (int i = 0; i < reps; ++i) { + nf.clone(); + } + } + + public void timeNewSimpleDateFormat(int reps) { + for (int i = 0; i < reps; ++i) { + new SimpleDateFormat(); + } + } + + public void timeClonedSimpleDateFormat(int reps) { + SimpleDateFormat sdf = new SimpleDateFormat(); + for (int i = 0; i < reps; ++i) { + sdf.clone(); + } + } + + // TODO: remove this from all examples when IDE plugins are ready + public static void main(String[] args) throws Exception { + CaliperMain.main(ExpensiveObjectsBenchmark.class, args); +// Runner.main(ExpensiveObjectsBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/FormatterBenchmark.java b/examples/src/main/java/examples/FormatterBenchmark.java new file mode 100644 index 0000000..b4a0541 --- /dev/null +++ b/examples/src/main/java/examples/FormatterBenchmark.java @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; +import java.util.Formatter; + +/** + * Compares Formatter against hand-written StringBuilder code. + */ +public class FormatterBenchmark extends SimpleBenchmark { + public void timeFormatter_NoFormatting(int reps) { + for (int i = 0; i < reps; i++) { + Formatter f = new Formatter(); + f.format("this is a reasonably short string that doesn't actually need any formatting"); + } + } + + public void timeStringBuilder_NoFormatting(int reps) { + for (int i = 0; i < reps; i++) { + StringBuilder sb = new StringBuilder(); + sb.append("this is a reasonably short string that doesn't actually need any formatting"); + } + } + + public void timeFormatter_OneInt(int reps) { + for (int i = 0; i < reps; i++) { + Formatter f = new Formatter(); + f.format("this is a reasonably short string that has an int %d in it", i); + } + } + + public void timeStringBuilder_OneInt(int reps) { + for (int i = 0; i < reps; i++) { + StringBuilder sb = new StringBuilder(); + sb.append("this is a reasonably short string that has an int "); + sb.append(i); + sb.append(" in it"); + } + } + + public void timeFormatter_OneString(int reps) { + for (int i = 0; i < reps; i++) { + Formatter f = new Formatter(); + f.format("this is a reasonably short string that has a string %s in it", "hello"); + } + } + + public void timeStringBuilder_OneString(int reps) { + for (int i = 0; i < reps; i++) { + StringBuilder sb = new StringBuilder(); + sb.append("this is a reasonably short string that has a string "); + sb.append("hello"); + sb.append(" in it"); + } + } + + public static void main(String[] args) throws Exception { + Runner.main(FormatterBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/IntModBenchmark.java b/examples/src/main/java/examples/IntModBenchmark.java new file mode 100644 index 0000000..55a119c --- /dev/null +++ b/examples/src/main/java/examples/IntModBenchmark.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; + +/** + * Measures several candidate implementations for mod(). + */ +@SuppressWarnings("SameParameterValue") +public class IntModBenchmark extends SimpleBenchmark { + private static final int M = (1 << 16) - 1; + + public int timeConditional(int reps) { + int dummy = 5; + for (int i = 0; i < reps; i++) { + dummy += Integer.MAX_VALUE + conditionalMod(dummy, M); + } + return dummy; + } + + private static int conditionalMod(int a, int m) { + int r = a % m; + return r < 0 ? r + m : r; + } + + public int timeDoubleRemainder(int reps) { + int dummy = 5; + for (int i = 0; i < reps; i++) { + dummy += Integer.MAX_VALUE + doubleRemainderMod(dummy, M); + } + return dummy; + } + + @SuppressWarnings("NumericCastThatLosesPrecision") // result of % by an int must be in int range + private static int doubleRemainderMod(int a, int m) { + return (int) ((a % m + (long) m) % m); + } + + public int timeRightShiftingMod(int reps) { + int dummy = 5; + for (int i = 0; i < reps; i++) { + dummy += Integer.MAX_VALUE + rightShiftingMod(dummy, M); + } + return dummy; + } + + @SuppressWarnings("NumericCastThatLosesPrecision") // must be in int range + private static int rightShiftingMod(int a, int m) { + long r = a % m; + return (int) (r + (r >> 63 & m)); + } + + public int timeLeftShiftingMod(int reps) { + int dummy = 5; + for (int i = 0; i < reps; i++) { + dummy += Integer.MAX_VALUE + leftShiftingMod(dummy, M); + } + return dummy; + } + + @SuppressWarnings("NumericCastThatLosesPrecision") // result of % by an int must be in int range + private static int leftShiftingMod(int a, int m) { + return (int) ((a + ((long) m << 32)) % m); + } + + public int timeWrongMod(int reps) { + int dummy = 5; + for (int i = 0; i < reps; i++) { + dummy += Integer.MAX_VALUE + dummy % M; + } + return dummy; + } + + // TODO: remove this from all examples when IDE plugins are ready + public static void main(String[] args) throws Exception { + Runner.main(IntModBenchmark.class, args); + } +} \ No newline at end of file diff --git a/examples/src/main/java/examples/ListIterationBenchmark.java b/examples/src/main/java/examples/ListIterationBenchmark.java new file mode 100644 index 0000000..a8cfb05 --- /dev/null +++ b/examples/src/main/java/examples/ListIterationBenchmark.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; +import java.util.AbstractList; +import java.util.List; + +/** + * Measures iterating through list elements. + */ +public class ListIterationBenchmark extends SimpleBenchmark { + + @Param({"0", "10", "100", "1000"}) + private int length; + + private List list; + private Object[] array; + + @Override protected void setUp() { + array = new Object[length]; + for (int i = 0; i < length; i++) { + array[i] = new Object(); + } + + list = new AbstractList() { + @Override public int size() { + return length; + } + + @Override public Object get(int i) { + return array[i]; + } + }; + } + + @SuppressWarnings({"UnusedDeclaration"}) // TODO: fix + public void timeListIteration(int reps) { + for (int i = 0; i < reps; i++) { + for (Object value : list) { + } + } + } + + @SuppressWarnings({"UnusedDeclaration"}) // TODO: fix + public void timeArrayIteration(int reps) { + for (int i = 0; i < reps; i++) { + for (Object value : array) { + } + } + } + + // TODO: remove this from all examples when IDE plugins are ready + public static void main(String[] args) throws Exception { + Runner.main(ListIterationBenchmark.class, args); + } +} \ No newline at end of file diff --git a/examples/src/main/java/examples/LoopingBackwardsBenchmark.java b/examples/src/main/java/examples/LoopingBackwardsBenchmark.java new file mode 100644 index 0000000..1e3d1ad --- /dev/null +++ b/examples/src/main/java/examples/LoopingBackwardsBenchmark.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; + +/** + * Testing the old canard that looping backwards is faster. + */ +public class LoopingBackwardsBenchmark extends SimpleBenchmark { + @Param({"2", "20", "2000", "20000000"}) int max; + + public int timeForwards(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + for (int j = 0; j < max; j++) { + dummy += j; + } + } + return dummy; + } + + public int timeBackwards(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + for (int j = max - 1; j >= 0; j--) { + dummy += j; + } + } + return dummy; + } + + public static void main(String[] args) throws Exception { + Runner.main(LoopingBackwardsBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/MessageDigestCreationBenchmark.java b/examples/src/main/java/examples/MessageDigestCreationBenchmark.java new file mode 100644 index 0000000..c9437bd --- /dev/null +++ b/examples/src/main/java/examples/MessageDigestCreationBenchmark.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; + +import java.security.MessageDigest; + +/** + * Times creating new MessageDigest instances. + */ +public class MessageDigestCreationBenchmark extends SimpleBenchmark { + // By default, just the "interesting ones". Also consider Adler32 and CRC32, + // but these are not guaranteed to be supported in all runtime environments. + @Param({"MD5", "SHA-1", "SHA-256", "SHA-512"}) + String algorithm; + + public void time(int reps) throws Exception { + // Change this to use a dummy if the results look suspicious. + for (int i = 0; i < reps; i++) { + MessageDigest.getInstance(algorithm); + } + } + + public static void main(String[] args) throws Exception { + Runner.main(MessageDigestCreationBenchmark.class, args); + } +} diff --git a/examples/src/main/java/examples/StringBuilderBenchmark.java b/examples/src/main/java/examples/StringBuilderBenchmark.java new file mode 100644 index 0000000..3c839d3 --- /dev/null +++ b/examples/src/main/java/examples/StringBuilderBenchmark.java @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package examples; + +import com.google.caliper.Param; +import com.google.caliper.Runner; +import com.google.caliper.SimpleBenchmark; + +/** + * Tests the performance of various StringBuilder methods. + */ +public class StringBuilderBenchmark extends SimpleBenchmark { + + @Param({"1", "10", "100"}) private int length; + + public void timeAppendBoolean(int reps) { + for (int i = 0; i < reps; ++i) { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < length; ++j) { + sb.append(true); + } + } + } + + public void timeAppendChar(int reps) { + for (int i = 0; i < reps; ++i) { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < length; ++j) { + sb.append('c'); + } + } + } + + public void timeAppendCharArray(int reps) { + char[] chars = "chars".toCharArray(); + for (int i = 0; i < reps; ++i) { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < length; ++j) { + sb.append(chars); + } + } + } + + public void timeAppendCharSequence(int reps) { + CharSequence cs = "chars"; + for (int i = 0; i < reps; ++i) { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < length; ++j) { + sb.append(cs); + } + } + } + + public void timeAppendDouble(int reps) { + double d = 1.2; + for (int i = 0; i < reps; ++i) { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < length; ++j) { + sb.append(d); + } + } + } + + public void timeAppendFloat(int reps) { + float f = 1.2f; + for (int i = 0; i < reps; ++i) { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < length; ++j) { + sb.append(f); + } + } + } + + public void timeAppendInt(int reps) { + int n = 123; + for (int i = 0; i < reps; ++i) { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < length; ++j) { + sb.append(n); + } + } + } + + public void timeAppendLong(int reps) { + long l = 123; + for (int i = 0; i < reps; ++i) { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < length; ++j) { + sb.append(l); + } + } + } + + public void timeAppendObject(int reps) { + Object o = new Object(); + for (int i = 0; i < reps; ++i) { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < length; ++j) { + sb.append(o); + } + } + } + + public void timeAppendString(int reps) { + String s = "chars"; + for (int i = 0; i < reps; ++i) { + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < length; ++j) { + sb.append(s); + } + } + } + + // TODO: remove this from all examples when IDE plugins are ready + public static void main(String[] args) throws Exception { + Runner.main(StringBuilderBenchmark.class, args); + } +} diff --git a/lib/gson-1.7.1.jar b/lib/gson-1.7.1.jar new file mode 100644 index 0000000..eb54274 Binary files /dev/null and b/lib/gson-1.7.1.jar differ diff --git a/lib/java-allocation-instrumenter-2.0.jar b/lib/java-allocation-instrumenter-2.0.jar new file mode 100644 index 0000000..8ea4193 Binary files /dev/null and b/lib/java-allocation-instrumenter-2.0.jar differ diff --git a/scripts/caliper b/scripts/caliper new file mode 100644 index 0000000..a25caf1 --- /dev/null +++ b/scripts/caliper @@ -0,0 +1,9 @@ +#!/bin/sh + +# rough + +export PATH=$PATH:$JAVA_HOME/bin +base=`dirname $0` +export ALLOCATION_JAR=$base/lib/allocation.jar +exec java -cp $base/lib/caliper-@VERSION@.jar:$ALLOCATION_JAR:$CLASSPATH com.google.caliper.Runner $* + diff --git a/tutorial/Tutorial.java b/tutorial/Tutorial.java new file mode 100644 index 0000000..625d387 --- /dev/null +++ b/tutorial/Tutorial.java @@ -0,0 +1,187 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tutorial; + +import com.google.caliper.Param; +import com.google.caliper.SimpleBenchmark; + +/** + * Caliper tutorial. To run the example benchmarks in this file: + * {@code CLASSPATH=... [caliper_home]/caliper tutorial.Tutorial.Benchmark1} + */ +public class Tutorial { + + /* + * We begin the Caliper tutorial with the simplest benchmark you can write. + * We'd like to know how efficient the method System.nanoTime() is. + * + * Notice: + * + * - We write a class that extends com.google.caliper.SimpleBenchmark. + * - It contains a public instance method whose name begins with 'time' and + * which accepts a single 'int reps' parameter. + * - The body of the method simply executes the code we wish to measure, + * 'reps' times. + * + * Example run: + * + * $ CLASSPATH=build/classes/test caliper tutorial.Tutorial.Benchmark1 + * [real-time results appear on this line] + * + * Summary report for tutorial.Tutorial$Benchmark1: + * + * Benchmark ns + * --------- --- + * NanoTime 233 + */ + public static class Benchmark1 extends SimpleBenchmark { + public void timeNanoTime(int reps) { + for (int i = 0; i < reps; i++) { + System.nanoTime(); + } + } + } + + /* + * Now let's compare two things: nanoTime() versus currentTimeMillis(). + * Notice: + * + * - We simply add another method, following the same rules as the first. + * + * Example run output: + * + * Benchmark ns + * ----------------- --- + * NanoTime 248 + * CurrentTimeMillis 118 + */ + public static class Benchmark2 extends SimpleBenchmark { + public void timeNanoTime(int reps) { + for (int i = 0; i < reps; i++) { + System.nanoTime(); + } + } + public void timeCurrentTimeMillis(int reps) { + for (int i = 0; i < reps; i++) { + System.currentTimeMillis(); + } + } + } + + /* + * Let's try iterating over a large array. This seems simple enough, but + * there is a problem! + */ + public static class Benchmark3 extends SimpleBenchmark { + private final int[] array = new int[1000000]; + + @SuppressWarnings("UnusedDeclaration") // IDEA tries to warn us! + public void timeArrayIteration_BAD(int reps) { + for (int i = 0; i < reps; i++) { + for (int ignoreMe : array) {} + } + } + } + + /* + * Caliper reported that the benchmark above ran in 4 nanoseconds. + * + * Wait, what? + * + * How can it possibly iterate over a million zeroes in 4 ns!? + * + * It is very important to sanity-check benchmark results with common sense! + * In this case, we're indeed getting a bogus result. The problem is that the + * Java Virtual Machine is too smart: it detected the fact that the loop was + * producing no actual result, so it simply compiled it right out. The method + * never looped at all. To fix this, we need to use a dummy result value. + * + * Notice: + * + * - We simply change the 'time' method from 'void' to any return type we + * wish. Then we return a value that can't be known without actually + * performing the work, and thus we defeat the runtime optimizations. + * - We're no longer timing *just* the code we want to be testing - our + * result will now be inflated by the (small) cost of addition. This is an + * unfortunate fact of life with microbenchmarking. In fact, we were + * already inflated by the cost of an int comparison, "i < reps" as it was. + * + * With this change, Caliper should report a much more realistic value, more + * on the order of an entire millisecond. + */ + public static class Benchmark4 extends SimpleBenchmark { + private final int[] array = new int[1000000]; + + public int timeArrayIteration_fixed(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + for (int doNotIgnoreMe : array) { + dummy += doNotIgnoreMe; + } + } + return dummy; // framework ignores this, but it has served its purpose! + } + } + + /* + * Now we'd like to know how various other *sizes* of arrays perform. We + * don't want to have to cut and paste the whole benchmark just to provide a + * different size. What we need is a parameter! + * + * When you run this benchmark the same way you ran the previous ones, you'll + * now get an error: "No values provided for benchmark parameter 'size'". + * You can provide the value requested at the command line like this: + * + * [caliper_home]/caliper tutorial.Tutorial.Benchmark5 -Dsize=100} + * + * You'll see output like this: + * + * Benchmark size ns + * -------------- ---- --- + * ArrayIteration 100 51 + * + * Now that we've parameterized our benchmark, things are starting to get fun. + * Try passing '-Dsize=10,100,1000' and see what happens! + * + * Benchmark size ns + * -------------- ---- ----------------------------------- + * ArrayIteration 10 7 | + * ArrayIteration 100 49 |||| + * ArrayIteration 1000 477 |||||||||||||||||||||||||||||| + * + */ + public static class Benchmark5 extends SimpleBenchmark { + @Param int size; // set automatically by framework + + private int[] array; // set by us, in setUp() + + @Override protected void setUp() { + // @Param values are guaranteed to have been injected by now + array = new int[size]; + } + + public int timeArrayIteration(int reps) { + int dummy = 0; + for (int i = 0; i < reps; i++) { + for (int doNotIgnoreMe : array) { + dummy += doNotIgnoreMe; + } + } + return dummy; + } + } +} -- cgit v1.2.3