15 files changed, 543 insertions, 0 deletions
diff --git a/crosperf/experiment_files/README b/crosperf/experiment_files/README
new file mode 100644
index 00000000..d9c96870
--- /dev/null
+++ b/crosperf/experiment_files/README
@@ -0,0 +1,34 @@
+To use these experiment files, replace the board, remote and images
+placeholders and run crosperf on them.
+
+Further information about crosperf:
+https://sites.google.com/a/google.com/chromeos-toolchain-team-home2/home/team-tools-and-scripts/crosperf-cros-image-performance-comparison-tool
+
+The final experiment file should look something like the following (but with 
+different actual values for the fields):
+
+board: lumpy
+remote: 123.45.67.089
+
+# Add images you want to test:
+my_image {
+  chromeos_image: /usr/local/chromeos/src/build/images/lumpy/chromiumos_test_image.bin
+}
+
+vanilla_image {
+   chromeos_root: /usr/local/chromeos
+   build: lumpy-release/R35-5672.0.0
+}
+
+# Paste experiment benchmarks here. Example, I pasted
+# page_cycler_v2.morejs here.
+
+# This experiment just runs a short autotest which measures the performance of
+# Telemetry's page_cycler_v2.morejs. In addition, it profiles
+
+perg_args: record -e cycles
+
+benchmark: page_cycler_v2.morejs {
+   suite: telemetry_Crosperf
+   iterations: 1
+}
diff --git a/crosperf/experiment_files/aes_perf.exp b/crosperf/experiment_files/aes_perf.exp
new file mode 100644
index 00000000..063c74be
--- /dev/null
+++ b/crosperf/experiment_files/aes_perf.exp
@@ -0,0 +1,21 @@
+# This experiment just runs a short autotest which measures the performance of
+# aes encryption.
+#
+# You should replace all the placeholders, marked by angle-brackets, with the
+# appropriate actual values.
+
+name: aes_example
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+benchmark: platform_AesThroughput {
+}
+
+# Replace the chromeos image below with the actual path to your test image.
+test_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
diff --git a/crosperf/experiment_files/bloat_perf.exp b/crosperf/experiment_files/bloat_perf.exp
new file mode 100644
index 00000000..14681778
--- /dev/null
+++ b/crosperf/experiment_files/bloat_perf.exp
@@ -0,0 +1,25 @@
+# This experiment just runs a short telemety autotest which measures
+# the performance of the page_cycler_v2.bloat test.
+#
+# You should replace all the placeholders, marked by angle-brackets, with the
+# appropriate actual values.
+
+name: bloat_perf_example
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+perf_args: record -e cycles
+
+benchmark: page_cycler_v2.bloat {
+	suite: telemetry_Crosperf
+	iterations:1
+}
+
+# Replace the chromeos image below with the actual path to your test image.
+test_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
diff --git a/crosperf/experiment_files/morejs_perf.exp b/crosperf/experiment_files/morejs_perf.exp
new file mode 100644
index 00000000..ebc54753
--- /dev/null
+++ b/crosperf/experiment_files/morejs_perf.exp
@@ -0,0 +1,25 @@
+# This experiment just runs a short telemety autotest which measures
+# the performance of the page_cycler_v2.morejs test.
+#
+# You should replace all the placeholders, marked by angle-brackets, with the
+# appropriate actual values.
+
+name: morejs_perf_example
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+perf_args: record -e cycles
+
+benchmark: page_cycler_v2.morejs {
+	suite: telemetry_Crosperf
+	iterations: 1
+}
+
+# Replace the chromeos image below with the actual path to your test image.
+test_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
diff --git a/crosperf/experiment_files/non-telemetry-tests.exp b/crosperf/experiment_files/non-telemetry-tests.exp
new file mode 100644
index 00000000..0ad1fe5c
--- /dev/null
+++ b/crosperf/experiment_files/non-telemetry-tests.exp
@@ -0,0 +1,31 @@
+# This example experiment file showa how to run some basic non-Telemetry
+# autotest tests.
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+name: non_telemetry_tests_example
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+benchmark: BootPerfServer {
+  test_name: BootPerfServer
+  iterations: 1
+}
+
+benchmark: bvt {
+  test_name: suite:bvt
+}
+
+benchmark: login_LoginSuccess {
+  test_name: login_LoginSuccess
+}
+
+# Replace the chromeos image below with the actual path to your test image.
+test_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
diff --git a/crosperf/experiment_files/official-image.exp b/crosperf/experiment_files/official-image.exp
new file mode 100644
index 00000000..bce7d6a3
--- /dev/null
+++ b/crosperf/experiment_files/official-image.exp
@@ -0,0 +1,41 @@
+# This example experiment file shows how to run a basic test, using
+# official images.
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+name: official_image_example
+
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+benchmark: canvasmark {
+  suite:telemetry_Crosperf
+  iterations: 1
+}
+
+
+# Replace <path-to-your-chroot-goes-here> with the actual directory path
+# to the top of your ChromimumOS chroot.
+first_official_image {
+  chromeos_root:<path-to-your-chroot-goes-here>
+  # Replace "latest-official" with the appropriate xbuddy version alias
+  # for the official image you want to use (see
+  # http://www.chromium.org/chromium-os/how-tos-and-troubleshooting/using-the-dev-server/xbuddy-for-devserver#TOC-XBuddy-Paths
+  # for xbuddy syntax).  
+  build: latest-official
+}
+
+second_official_image {
+  # Replace <path-to-your-chroot-goes-here> with actual path.
+  chromeos_root:<path-to-your-chroot-goes-here>
+  # Replace "lumpy-release/R35-5672.0.0" with the official image you want
+  # to use.
+  build:lumpy-release/R35-5672.0.0
+}
+
+
diff --git a/crosperf/experiment_files/page_cycler.exp b/crosperf/experiment_files/page_cycler.exp
new file mode 100644
index 00000000..6cb6166d
--- /dev/null
+++ b/crosperf/experiment_files/page_cycler.exp
@@ -0,0 +1,28 @@
+# This experiment file shows how to run all of the Telemetry
+# page_cycler tests.
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+name: all_page_cyclers_example
+board: <your-board-goes-here>
+
+# Note: You can specify multiple remotes, to run your tests in
+# parallel on multiple machines. e.g. "remote: test-machine-1.com
+# test-machine2.come test-machine3.com"
+
+remote: <your-remote-goes-here>
+
+
+# NOTE: all_pagecyclers is a Crosperf alias that will cause all of the 
+# Telemetry page_cycler benchmark tests to be run. 
+benchmark: all_pagecyclers {
+	suite: telemetry_Crosperf
+	iterations: 2
+}
+
+# Replace the chromeos image below with the actual path to your test
+# image.
+test_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
diff --git a/crosperf/experiment_files/page_cycler_perf.exp b/crosperf/experiment_files/page_cycler_perf.exp
new file mode 100644
index 00000000..cd661737
--- /dev/null
+++ b/crosperf/experiment_files/page_cycler_perf.exp
@@ -0,0 +1,45 @@
+# This experiment profiles some of the Telemetry page cycler tests,
+# uisng 'perf' on the remotes to get performance profiles.
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+name: aes_example
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+perf_args: record -e cycles,instructions
+
+benchmark: page_cycler_v2.morejs {
+	suite: telemetry_Crosperf
+	iterations: 10
+}
+
+benchmark: page_cycler_v2.bloat {
+	suite: telemetry_Crosperf
+	iterations: 10
+}
+
+benchmark: page_cycler_v2.dhtml {
+	suite: telemetry_Crosperf
+	iterations: 10
+}
+
+benchmark: page_cycler_v2.intl_ar_fa_he {
+	suite: telemetry_Crosperf
+	iterations: 10
+}
+
+benchmark: page_cycler_v2.moz {
+	suite: telemetry_Crosperf
+	iterations: 10
+}
+
+# Replace the chromeos image below with the actual path to your test image.
+test_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
diff --git a/crosperf/experiment_files/telemetry-crosperf-suites.exp b/crosperf/experiment_files/telemetry-crosperf-suites.exp
new file mode 100644
index 00000000..2caa588d
--- /dev/null
+++ b/crosperf/experiment_files/telemetry-crosperf-suites.exp
@@ -0,0 +1,54 @@
+# This example experiment file shows how to invoke sets of tests (a
+# set is a group of tests that can be invoked by a single alias).
+# There are currently three sets defined for crosperf_Telemetry:
+# all_perfv2, all_pagecyclers, and all_toolchain_perf.
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+
+name: telemetry_crosperf_suites_example
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+# The example below will run all the benchmarks in the perf_v2 suite.
+# The exact list of benchmarks that will be run can be seen in
+# crosperf/experiment_factory.py
+benchmark: all_perfv2 {
+  suite:telemetry_Crosperf
+  iterations: 2
+}
+
+# The example below will run all the Telemetry page_cycler benchmarks.
+# The exact list of benchmarks that will be run can be seen in
+# crosperf/experiment_factory.py
+benchmark: all_pagecyclers {
+  suite:telemetry_Crosperf
+  iterations: 1
+}
+
+# The example below will run all the Telemetry page_cycler benchmarks.
+# The exact list of benchmarks that will be run can be seen in
+# crosperf/experiment_factory.py
+benchmark: all_toolchain_perf {
+  suite:telemetry_Crosperf
+  iterations: 1
+}
+
+# Replace the chromeos image below with the actual path to your test image.
+test_image_1 {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
+
+# Replace the chromeos image below with the actual path to your second
+# test image (if desired).
+new_image {
+  chromeos_image:<path-to-your-other-chroot-goes-here>/src/build/images/<board-goes-here>/latest/chromiumos_test_image.bin
+}
+
+
+
diff --git a/crosperf/experiment_files/telemetry-crosperf-with-external-chrome-src.exp b/crosperf/experiment_files/telemetry-crosperf-with-external-chrome-src.exp
new file mode 100644
index 00000000..517c13f1
--- /dev/null
+++ b/crosperf/experiment_files/telemetry-crosperf-with-external-chrome-src.exp
@@ -0,0 +1,31 @@
+# This example experiment file showings how to specify an external
+# chrome source tree (rather than using the one inside the chroot).
+# The Telemetry tests will be run from the external Chrome source
+# tree.
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+name: telemetry_crosperf_external_src_example
+
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+benchmark: octane {
+    suite: telemetry_Crosperf
+    iterations: 1
+}
+
+# Replace the chromeos image below with the actual path to your test imnage.
+test_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+  # Replace '/usr/local/google/chrome-top' with the path to the
+  # top of your Chrome source tree. From that directory
+  # "./src/tools/perf/run_benchmark" should be a valid file path.
+  chrome_src:/usr/local/google/chrome-top
+}
+
diff --git a/crosperf/experiment_files/telemetry-crosperf-with-profiler.exp b/crosperf/experiment_files/telemetry-crosperf-with-profiler.exp
new file mode 100644
index 00000000..4c2b88fc
--- /dev/null
+++ b/crosperf/experiment_files/telemetry-crosperf-with-profiler.exp
@@ -0,0 +1,35 @@
+# This example experiment file shows how to invoke the profiler (via
+# the perf_args above the benchmark).
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+
+name: telemetry_crosperf_profiler_example
+
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+# Below is the line that causes the profiler to run.  Currently the
+# only profiler option is running 'perf' on the remote machine.  If
+# you want you can replace 'record' with 'stat'.  You would also need
+# to change the other args accordingly.  Crosperf automatically
+# inserts a '-a' if you use 'record' for you perf_args.  The results
+# of the perf run (perf.data and perf.report files) will be available
+# with the rest of the Crosperf results.
+perf_args: record -e cycles,instructions
+
+benchmark: page_cycler_v2.dhtml {
+    suite: telemetry_Crosperf
+    iterations: 1
+}
+
+# Replace the chromeos image below with the actual path to your test imnage.
+test_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
+
diff --git a/crosperf/experiment_files/telemetry-crosperf.exp b/crosperf/experiment_files/telemetry-crosperf.exp
new file mode 100644
index 00000000..111001d4
--- /dev/null
+++ b/crosperf/experiment_files/telemetry-crosperf.exp
@@ -0,0 +1,32 @@
+# This example experiment file shows how to run a Telemetry test,
+# using autotest (via "suite: telemetry_Crosperf").  This runs the
+# Telemetry's "run_benchmark" for the specified test.
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+name: basic_telemetry_crosperf_example
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+# Replace "octane" below with the name of the Telemetry benchmark you 
+# want to run.
+benchmark: octane {
+    suite: telemetry_Crosperf
+    iterations: 1
+}
+
+# NOTE: You must specify at least one image; you may specify more than one.
+# Replace <path-to-your-chroot-goes-here> and <board-goes-here> below.
+vanilla_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/vanilla-image/chromiumos_test_image.bin
+}
+
+# Replace the chromeos image below with the actual path to your test image.
+test_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
diff --git a/crosperf/experiment_files/telemetry-without-autotest.exp b/crosperf/experiment_files/telemetry-without-autotest.exp
new file mode 100644
index 00000000..ce3f207e
--- /dev/null
+++ b/crosperf/experiment_files/telemetry-without-autotest.exp
@@ -0,0 +1,31 @@
+# This example experiment file shows how to run a Telemetry test
+# directly, bypassing autotest.  This runs the "run_measurement"
+# script.  You need to supply both the name of the Telemetry test and
+# the page_set (via the test_args argument).
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+name: telemetry_without_autotest_example
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+# Replace "page_cycler_dhtml" below with the name of the Telemetry test
+# that you want run_measurement to run.  Also replace the page set below
+# (in the test_args field) with the appropriate page set for your test.
+# N.B.  The key to running telemetry without autotest is the 'suite' field.
+# Make sure your suite is 'telemtry', NOT 'telemetry_Crosperf'.
+benchmark: page_cycler_dhtml {
+    suite: telemetry 
+    iterations: 1
+    test_args: ./page_sets/page_cycler/dhtml.json
+}
+
+# Replace the chromeos image below with the actual path to your test image.
+test_image {
+  chromeos_image:<path-to-your-chroot>/src/build/images/<board>/test-image/chromiumos_test_image.bin
+}
diff --git a/crosperf/experiment_files/telemetry_perf_perf b/crosperf/experiment_files/telemetry_perf_perf
new file mode 100755
index 00000000..acdf96d0
--- /dev/null
+++ b/crosperf/experiment_files/telemetry_perf_perf
@@ -0,0 +1,77 @@
+#!/bin/bash
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Script for generating and running telemetry benchmarkes via crosperf with
+# different perf command lines in order to measure the impact of the perf
+# commands on performance. Crosperf cannot run the same benchmark multiple
+# times, so this script runs crosperf multpilpe times instead. Unfortunately,
+# this means you must compare the results yourself.
+#
+# Perf will run for the entire benchmark run, so results should be interpreted
+# in that context. i.e, if this shows a 3% overhead for a particular perf
+# command, that overhead would only be seen during the 2 seconds of measurement
+# during a Chrome OS Wide Profiling collection.
+set -e
+
+board=xxx #<you-board-here>
+remote=xxx #<your-remote-here>
+iterations=5
+chromeos_root=~/chromiumos
+chrome_src=~/chromium
+
+
+function GenerateExperiment() {
+  local perf_args="${1:+perf_args: $1}"
+  local track="$2"  # stable, beta, dev
+
+  cat <<_EOF
+$perf_args
+benchmark: page_cycler_v2.typical_25 {
+    suite: telemetry_Crosperf
+}
+
+$track {
+  build: latest-$track
+}
+_EOF
+}
+
+function RunExperiment() {
+  local name="$1"
+  local perf_command="$2"
+  GenerateExperiment "$perf_command" "stable" > /tmp/crosperf.exp
+  ./crosperf /tmp/crosperf.exp \
+    --name telemetry_perf_perf_${name} \
+    --board="${board}" \
+    --remote="${remote}" \
+    --iterations="${iterations}" \
+    --chromeos_root="${chromeos_root}" \
+    --chrome_src="${chrome_src}" \
+    --rerun=true \
+    --use_file_locks=true \
+    --locks_dir=/tmp/crosperf.locks
+}
+
+if [ "$board" = "xxx" -o "$remote" = "xxx" ]; then
+  echo "Please set board and remote at the top of this script before running."
+  exit -1
+fi
+
+
+# Note that "-a" is automatically inserted in the perf command line.
+
+# Control: No profiling.
+RunExperiment 'control' ''
+# This is our baseline standard 'cycles' perf command.
+RunExperiment 'cycles.flat' \
+  'record -e cycles -c 1000003'
+# Callgraph profiling.
+RunExperiment 'cycles.callgraph' \
+  'record -g -e cycles -c 4000037'
+# Memory bandwidth profiling. As a perf stat command, we expect imperceptible
+# overhead.
+RunExperiment 'memory.bandwidth' \
+  'stat -e cycles -e instructions -e uncore_imc/data_reads/ -e uncore_imc/data_writes/ -e cpu/event=0xD0,umask=0x11,name=MEM_UOPS_RETIRED-STLB_MISS_LOADS/ -e cpu/event=0xD0,umask=0x12,name=MEM_UOPS_RETIRED-STLB_MISS_STORES/'
+
diff --git a/crosperf/experiment_files/trybot-image.exp b/crosperf/experiment_files/trybot-image.exp
new file mode 100644
index 00000000..a261e08c
--- /dev/null
+++ b/crosperf/experiment_files/trybot-image.exp
@@ -0,0 +1,33 @@
+# This example experiment shows how to run a basic test, using a
+# (previously made) trybot image.
+
+#
+# You should replace all the placeholders, marked by angle-brackets,
+# with the appropriate actual values.
+
+name: trybot_example
+board: <your-board-goes-here>
+
+# Note:  You can specify multiple remotes, to run your tests in parallel on
+# multiple machines. e.g. "remote: test-machine-1.com test-machine2.come
+# test-machine3.com"
+remote: <your-remote-goes-here>
+
+
+benchmark: canvasmark {
+  suite:telemetry_Crosperf
+  iterations: 1
+}
+
+
+# Replace <path-to-your-chroot-goes-here> with the actual directory path
+# to the top of your ChromimumOS chroot.
+trybot_image {
+  chromeos_root:<path-to-your-chroot-goes-here>
+  # Replace "trybot-lumpy-paladin/R34-5417.0.0-b1506" with the name of the
+  # trybot image that you wish to use.  You can find this by going to the
+  # trybot build log, going to the 'Report' stage, and looking for 'Build
+  # Artifacts' at the bottom. You can extract the trybot image name from that.
+  build:trybot-lumpy-paladin/R34-5417.0.0-b1506
+}
+