diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/binder/benchmarks/binderAddInts.cpp | 40 | ||||
| -rwxr-xr-x | tests/workloads/chromefling.sh | 160 | ||||
| -rwxr-xr-x | tests/workloads/defs.sh | 53 | ||||
| -rwxr-xr-x | tests/workloads/powerave.py | 46 | ||||
| -rwxr-xr-x | tests/workloads/pwrsummary.sh | 235 | ||||
| -rwxr-xr-x | tests/workloads/pwrtest.sh | 362 | ||||
| -rwxr-xr-x | tests/workloads/recentfling.sh | 8 | ||||
| -rwxr-xr-x | tests/workloads/systemapps.sh | 23 | ||||
| -rwxr-xr-x | tests/workloads/youtube.sh | 139 |
9 files changed, 1041 insertions, 25 deletions
diff --git a/tests/binder/benchmarks/binderAddInts.cpp b/tests/binder/benchmarks/binderAddInts.cpp index ff7ea079..f1b2e727 100644 --- a/tests/binder/benchmarks/binderAddInts.cpp +++ b/tests/binder/benchmarks/binderAddInts.cpp @@ -36,6 +36,7 @@ #include <cerrno> #include <grp.h> #include <iostream> +#include <iomanip> #include <libgen.h> #include <time.h> #include <unistd.h> @@ -88,8 +89,23 @@ class AddIntsService : public BBinder int cpu_; }; +struct Duration { + double value; + friend std::ostream& operator<<(std::ostream& stream, const Duration& d) { + static const char *SUFFIXES[] = {"s", "ms", "us", "ns"}; + size_t suffix = 0; + double temp = d.value; + while (temp < .1 && suffix < 3) { + temp *= 1000; + suffix++; + } + stream << temp << SUFFIXES[suffix]; + return stream; + } +}; + // File scope function prototypes -static void server(void); +static bool server(void); static void client(void); static void bindCPU(unsigned int cpu); static ostream &operator<<(ostream &stream, const String16& str); @@ -196,7 +212,7 @@ int main(int argc, char *argv[]) return 0; default: // Parent - server(); + if (!server()) { break; } // Wait for all children to end do { @@ -219,7 +235,7 @@ int main(int argc, char *argv[]) return 0; } -static void server(void) +static bool server(void) { int rv; @@ -230,10 +246,12 @@ static void server(void) new AddIntsService(options.serverCPU))) != 0) { cerr << "addService " << serviceName << " failed, rv: " << rv << " errno: " << errno << endl; + return false; } // Start threads to handle server work proc->startThreadPool(); + return true; } static void client(void) @@ -248,12 +266,17 @@ static void client(void) // Attach to service sp<IBinder> binder; - do { + for (int i = 0; i < 3; i++) { binder = sm->getService(serviceName); if (binder != 0) break; cout << serviceName << " not published, waiting..." << endl; usleep(500000); // 0.5 s - } while(true); + } + + if (binder == 0) { + cout << serviceName << " failed to publish, aborting" << endl; + return; + } // Perform the IPC operations for (unsigned int iter = 0; iter < options.iterations; iter++) { @@ -298,9 +321,10 @@ static void client(void) } // Display the results - cout << "Time per iteration min: " << min - << " avg: " << (total / options.iterations) - << " max: " << max + cout << fixed << setprecision(2) + << "Time per iteration min: " << Duration{min} + << " avg: " << Duration{total / options.iterations} + << " max: " << Duration{max} << endl; } diff --git a/tests/workloads/chromefling.sh b/tests/workloads/chromefling.sh new file mode 100755 index 00000000..a86f274b --- /dev/null +++ b/tests/workloads/chromefling.sh @@ -0,0 +1,160 @@ +# +# Script to start 3 chrome tabs, fling each of them, repeat +# For each iteration, Total frames and janky frames are reported. +# +# Options are described below. +# +iterations=10 +startapps=1 +capturesystrace=0 +waittime=4 +app=chrome + +function processLocalOption { + ret=0 + case "$1" in + (-N) startapps=0;; + (-A) unset appList;; + (-L) appList=$2; shift; ret=1;; + (-T) capturesystrace=1;; + (-W) waittime=$2; shift; ret=1;; + (*) + echo "$0: unrecognized option: $1" + echo; echo "Usage: $0 [options]" + echo "-A : use all known applications" + echo "-L applist : list of applications" + echo " default: $appList" + echo "-N : no app startups, just fling" + echo "-g : generate activity strings" + echo "-i iterations" + echo "-T : capture systrace on each iteration" + echo "-d device : device type (shamu, volantis, bullhead,...)" + exit 1;; + esac + return $ret +} + +CMDDIR=$(dirname $0 2>/dev/null) +CMDDIR=${CMDDIR:=.} +. $CMDDIR/defs.sh + +case $DEVICE in +(hammerhead) + flingtime=300 + downCount=2 + upCount=6 + UP="70 400 70 100 $flingtime" + DOWN="70 100 70 400 $flingtime";; +(shamu) + flingtime=100 + downCount=2 + upCount=2 + UP="700 1847 700 400 $flingtime" + DOWN="700 400 700 1847 $flingtime";; +(angler) + flingtime=150 + downCount=4 + upCount=3 + UP="500 1200 500 550 $flingtime" + DOWN="500 550 500 1200 $flingtime";; +(bullhead|volantis) + flingtime=200 + downCount=5 + upCount=5 + UP="500 1400 500 400 $flingtime" + DOWN="500 400 500 1400 $flingtime";; +(*) + echo "Error: No display information available for $DEVICE" + exit 1;; +esac + +function swipe { + count=0 + while [ $count -lt $2 ] + do + doSwipe $1 + ((count=count+1)) + done + sleep 1 +} + +cur=1 +frameSum=0 +jankSum=0 +latency90Sum=0 +latency95Sum=0 +latency99Sum=0 + +doKeyevent HOME +sleep 0.5 +resetJankyFrames $(getPackageName $app) + +while [ $cur -le $iterations ] +do + if [ $capturesystrace -gt 0 ]; then + ${ADB}atrace --async_start -z -c -b 16000 freq gfx view idle sched + fi + t=$(startActivity $app) + sleep $waittime + swipe "$UP" $upCount + + t=$(startActivity $app) + sleep $waittime + swipe "$UP" $upCount + + t=$(startActivity $app) + sleep $waittime + swipe "$UP" $upCount + + doKeyevent BACK + sleep $waittime + swipe "$DOWN" $downCount + + doKeyevent BACK + sleep $waittime + swipe "$DOWN" $downCount + + doKeyevent BACK + sleep 0.5 + + if [ $capturesystrace -gt 0 ]; then + ${ADB}atrace --async_dump -z -c -b 16000 freq gfx view idle sched > trace.${cur}.out + fi + doKeyevent HOME + sleep 0.5 + + set -- $(getJankyFrames $(getPackageName $app)) + totalDiff=$1 + jankyDiff=$2 + latency90=$3 + latency95=$4 + latency99=$5 + if [ ${totalDiff:=0} -eq 0 ]; then + echo Error: could not read frame info with \"dumpsys gfxinfo\" + exit 1 + fi + + ((frameSum=frameSum+totalDiff)) + ((jankSum=jankSum+jankyDiff)) + ((latency90Sum=latency90Sum+latency90)) + ((latency95Sum=latency95Sum+latency95)) + ((latency99Sum=latency99Sum+latency99)) + if [ "$totalDiff" -eq 0 ]; then + echo Error: no frames detected. Is the display off? + exit 1 + fi + ((jankPct=jankyDiff*100/totalDiff)) + resetJankyFrames $(getPackageName $app) + + + echo Frames: $totalDiff latency: $latency90/$latency95/$latency99 Janks: $jankyDiff\(${jankPct}%\) + ((cur=cur+1)) +done +doKeyevent HOME +((aveJankPct=jankSum*100/frameSum)) +((aveJanks=jankSum/iterations)) +((aveFrames=frameSum/iterations)) +((aveLatency90=latency90Sum/iterations)) +((aveLatency95=latency95Sum/iterations)) +((aveLatency99=latency99Sum/iterations)) +echo AVE: Frames: $aveFrames latency: $aveLatency90/$aveLatency95/$aveLatency99 Janks: $aveJanks\(${aveJankPct}%\) diff --git a/tests/workloads/defs.sh b/tests/workloads/defs.sh index a2b71387..0f50fe16 100755 --- a/tests/workloads/defs.sh +++ b/tests/workloads/defs.sh @@ -16,15 +16,16 @@ youtubeActivity='com.google.android.youtube/com.google.android.apps.youtube.app. cameraActivity='com.google.android.GoogleCamera/com.android.camera.CameraActivity' playActivity='com.android.vending/com.google.android.finsky.activities.MainActivity' feedlyActivity='com.devhd.feedly/com.devhd.feedly.Main' -photosActivity='com.google.android.apps.plus/com.google.android.apps.photos.phone.PhotosHomeActivity' +photosActivity='com.google.android.apps.photos/com.google.android.apps.photos.home.HomeActivity' mapsActivity='com.google.android.apps.maps/com.google.android.maps.MapsActivity' calendarActivity='com.google.android.calendar/com.android.calendar.AllInOneActivity' earthActivity='com.google.earth/com.google.earth.EarthActivity' -calculatorActivity='com.android.calculator2/com.android.calculator2.Calculator' +calculatorActivity='com.google.android.calculator/com.android.calculator2.Calculator' sheetsActivity='com.google.android.apps.docs.editors.sheets/com.google.android.apps.docs.app.NewMainProxyActivity' docsActivity='com.google.android.apps.docs.editors.docs/com.google.android.apps.docs.app.NewMainProxyActivity' operaActivity='com.opera.mini.native/com.opera.mini.android.Browser' firefoxActivity='org.mozilla.firefox/org.mozilla.firefox.App' +suntempleActivity='com.BrueComputing.SunTemple/com.epicgames.ue4.GameActivity' homeActivity='com.google.android.googlequicksearchbox/com.google.android.launcher.GEL' function showUsage { @@ -97,6 +98,18 @@ else isOnDevice=0 fi +if [ $isOnDevice -gt 0 ]; then + case "$DEVICE" in + (bullhead|angler) + if ! echo $$ > /dev/cpuset/background/tasks; then + echo Could not put PID $$ in background + fi + ;; + (*) + ;; + esac +fi + # default values if not set by options or calling script appList=${appList:=$dfltAppList} savetmpfiles=${savetmpfiles:=0} @@ -109,7 +122,9 @@ ADB=${ADB:=""} output=${output:="./out"} # clear the output file -> $output +if [ -f $output ]; then + > $output +fi # ADB commands AM_FORCE_START="${ADB}am start -W -S" @@ -162,6 +177,7 @@ function log2msec { in=$1 in=${in:=0.0} set -- $(echo $in | tr . " ") + # shell addition via (( )) doesn't like leading zeroes in msecs # field so remove leading zeroes msecfield=$(expr 0 + $2) @@ -209,7 +225,7 @@ function getEndTime { function resetJankyFrames { _gfxapp=$1 - _gfxapp=${app:="com.android.systemui"} + _gfxapp=${_gfxapp:="com.android.systemui"} ${ADB}dumpsys gfxinfo $_gfxapp reset 2>&1 >/dev/null } @@ -256,14 +272,21 @@ function checkForDirectReclaim { } function startInstramentation { + _iter=$1 + _iter=${_iter:=0} + enableAtrace=$2 + enableAtrace=${enableAtrace:=1} # Called at beginning of loop. Turn on instramentation like atrace vout start instramentation $(date) echo =============================== >> $output - echo Before iteration >> $output + echo Before iteration $_iter >> $output echo =============================== >> $output ${ADB}cat /proc/meminfo 2>&1 >> $output ${ADB}dumpsys meminfo 2>&1 >> $output - if [ "$user" = root ]; then + if [ "$DEVICE" = volantis ]; then + ${ADB}cat /d/nvmap/iovmm/procrank 2>&1 >> $output + fi + if [ "$user" = root -a $enableAtrace -gt 0 ]; then vout ${ADB}atrace -b 32768 --async_start $tracecategories ${ADB}atrace -b 32768 --async_start $tracecategories >> $output echo >> $output @@ -271,14 +294,15 @@ function startInstramentation { } function stopInstramentation { - if [ "$user" = root ]; then + enableAtrace=$1 + enableAtrace=${enableAtrace:=1} + if [ "$user" = root -a $enableAtrace -gt 0 ]; then vout ${ADB}atrace --async_stop ${ADB}atrace --async_stop > /dev/null fi } function stopAndDumpInstramentation { - # Called at beginning of loop. Turn on instramentation like atrace vout stop instramentation $(date) echo =============================== >> $output echo After iteration >> $output @@ -300,9 +324,9 @@ function stopAndDumpInstramentation { python $UNCOMPRESS $tmpTrace >> $traceout rm -f $tmpTrace else - ${ADB}atrace $zarg -b 32768 --async_dump >> $traceout + ${ADB}atrace -b 32768 --async_dump > $traceout fi - vout ${ADB}atrace $zarg --async_dump + vout ${ADB}atrace $zarg -b 32768 --async_dump vout ${ADB}atrace --async_stop ${ADB}atrace --async_stop > /dev/null fi @@ -375,7 +399,14 @@ function checkActivity { function doSwipe { vout ${ADB}input swipe $* - ${ADB}input swipe $* + ${ADB}nice input swipe $* +} + +function doText { + echo $* > ./tmpOutput + vout ${ADB}input text \"$*\" + ${ADB}input text "$(cat ./tmpOutput)" + rm -f ./tmpOutput } function doTap { diff --git a/tests/workloads/powerave.py b/tests/workloads/powerave.py new file mode 100755 index 00000000..4d786b97 --- /dev/null +++ b/tests/workloads/powerave.py @@ -0,0 +1,46 @@ +#!/usr/bin/python + +import sys +import getopt + +def usage(): + print "powersum.py [OPTIONS] HZ VOLTAGE [FILE]" + print "OPTIONS: " + print "-o OFFSET: subtract OFFSET from all data points" + print "\nHZ: samples per second in FILE or stdin" + sys.exit(0) + +offset = 0.0 +voltage = 4.3 + +parsedargv,argvrem = getopt.getopt(sys.argv[1:], "vo:w:l:h", ["help"]) +for o,a in parsedargv: + if o == '-o': offset = float(a) + if o == '-h' or o == '--help': usage() + +hz = float(argvrem[0]) +voltage = float(argvrem[1]) +if len(argvrem) > 1: + f = open(argvrem[2], "r") +else: + f = sys.stdin + +totalpower = 0.0 +samplectr = 0 + +for line in f: + try: + val = float(line.split(" ")[1]) # xxx take 2nd arg in line + val -= offset + except: + print "Can't parse data line, did you remember the timestamp?" + print "data was: %s" % line + sys.exit(1) + + samplectr+=1 + totalpower += val/hz + +avecurrent = totalpower * hz *1000 / samplectr +avepower = avecurrent * voltage + +print "%.3f %.3f" % (avecurrent, avepower) diff --git a/tests/workloads/pwrsummary.sh b/tests/workloads/pwrsummary.sh new file mode 100755 index 00000000..3d3aeb8a --- /dev/null +++ b/tests/workloads/pwrsummary.sh @@ -0,0 +1,235 @@ +# print summary of output generated by pwrtest.sh +# +# default results directories are <device>-<date>[-experiment]. By default +# match any device and the year 201*. +# +# Examples: +# +# - show output for all bullhead tests in july 2015: +# ./pwrsummary.sh -r "bh-201507*" +# +# - generate CSV file for import into spreadsheet: +# ./pwrsummary.sh -o csv +# + +CMDDIR=$(dirname $0 2>/dev/null) +CMDDIR=${CMDDIR:=.} +cd $CMDDIR +CMDDIR=$(pwd) +cd - +POWERAVE="python $CMDDIR/powerave.py" + +defaultPattern="*-201*" +defaultVoltage=4.3 +defaultFrequency=5 + +function Usage { + echo "$0 [-o format] [-v voltage] [-h freq] [-f resultsDirectories]" +} + +while [ $# -gt 0 ] +do + case "$1" in + (-o) format=$2; shift;; + (-v) voltage=$2; shift;; + (-h) hz=$2; shift;; + (-r) testResults="$2"; shift;; + (--help) Usage; exit 0;; + (--) shift; break;; + (*) + echo Unknown option: $1 + Usage + exit 1;; + esac + shift +done + +testResults=${testResults:=$defaultPattern} +voltage=${voltage:=$defaultVoltage} +hz=${hz:=$defaultFrequency} + +function printHeader { + workload=$1 + units="unknown" + case $workload in + (suntemple|shadowgrid2) + units="FPS";; + (recentfling|youtube|chrome) + units="FPS from app point of view: 1/(90th percentile render time)";; + (sysapps) + units="App start/switch per second";; + esac + + echo "Performance unit for $workload is: $units" + if [ "$format" = csv ]; then + printf "%s,%s,%s,%s,%s,%s,%s,%s,%s\n" " " build min ave max net-mA@${voltage}v base-mW net-mW perf/W + else + printf "%-30s %-8s %12.12s %12.12s %12.12s %12.12s %12.12s %12.12s %12.12s\n" " " build min ave max net-mA@${voltage}v base-mW net-mW perf/W + fi +} + +function average { + awk 'BEGIN { count=0; sum=0; max=-1000000000; min=1000000000; } + { + cur = $1; + sum = sum + cur; + if (cur > max) max = cur; + if (cur < min) min = cur; + count++; + } + + END { + if (count > 0) { + ave = sum / count; + printf "%.2f %.2f %.2f\n", min, ave, max; + } + }' +} + +function hwuiOutputParser { + # Stats since: 60659316905953ns + # Total frames rendered: 150 + # Janky frames: 89 (59.33%) + # 90th percentile: 23ms + # 95th percentile: 27ms + # 99th percentile: 32ms + # Number Missed Vsync: 0 + # Number High input latency: 0 + # Number Slow UI thread: 0 + # Number Slow bitmap uploads: 12 + # Number Slow draw: 89 + # use with "stdbuf -o0 " to disable pipe buffering + # stdbuf -o0 adb shell /data/hwuitest shadowgrid2 400 | stdbuf -o0 ./hwuitestfilter.sh | tee t.csv + sed -e 's/ns//' -e 's/[\(\)%]/ /g' | awk ' + BEGIN { startTime=0; lastTime=0; } + /^Stats since:/ { + curTime = $3; + if (startTime == 0) { + startTime = curTime; + } + if (lastTime) { + interval = curTime - lastTime; + fps = totalFrames*1000000000 / interval; + diffTime = curTime - startTime; + printf "%.2f, %.2f, ",diffTime/1000000, fps; + } + } + /^Total frames/ { totalFrames=$4; } + /^Janky frames:/ { + if (lastTime) { + printf "%.2f\n",$4; lastTime=curTime; + } + lastTime = curTime; + }' +} + +function sysappOutputParser { + awk ' + BEGIN { fmt=0; count=0; sum=0; } + /^App/ { + if (count != 0) { + if (fmt > 2) printf "Ave: %0.2fms\n", sum/count; + else printf " %0.2f\n", sum/count; + count = 0; + sum = 0; + } + } + /^[a-z]/ { val=$2; if (val != 0) { count++; sum+=val; } } + /^Iteration/ { if (fmt > 2) printf "%s : ", $0; else if (fmt) printf "%d ", $2; } + ' +} + +function calcPerfData { + testdir=$1 + workload=$2 + baselineCurrent=$3 + baselinePower=$4 + + file=${workload}.out + powerfile=${workload}-power.out + build="$(cat build 2>/dev/null)" + build=${build:="Unknown"} + + lines=$(wc -l $file 2>/dev/null | cut -f1 -d\ ) + + if [ ${lines:=0} -eq -0 ]; then + # No performance data captured + if [ "$format" = csv ]; then + printf "%s,%s,%s\n" $testdir "$build" "no data" + else + printf "%-30s %-8s %12.12s\n" $testdir "$build" "no data" + fi + return 1 + fi + + set -- $($POWERAVE $hz $voltage $powerfile) + current=$(echo $1 $baselineCurrent | awk '{ printf "%.2f", $1-$2; }') + power=$(echo $2 $baselinePower | awk '{ printf "%.2f", $1-$2; }') + + case $workload in + (idle) + set -- 0 0 0 + ;; + (suntemple) + # units are fps + set -- $(grep "FPS average" $file | sed 's/^.*seconds for a //' | awk '{ print $1; }' | average) + ;; + (recentfling|youtube|chrome) + # units are ms, so need to convert to app/ms + set -- $(grep ^Frames: $file | tr "/" " " | awk '{ print $4; }' | average | awk '{ printf "%.3f %.3f %.3f\n", 1000/$3, 1000/$2, 1000/$1;}' ) + ;; + (sysapps) + # units are ms, so need to convert to app/ms + set -- $(cat $file | sysappOutputParser | average | awk '{ printf "%.3f %.3f %.3f\n", 1000/$3, 1000/$2, 1000/$1;}' ) + ;; + (shadowgrid2) + # units are fps + set -- $(cat $file | hwuiOutputParser | tr ',' ' ' | awk '{print $2;}' | average) + ;; + esac + + minperf=$1 + aveperf=$2 + maxperf=$3 + perfPerWatt=$(echo $aveperf $power | awk '{ if ($2) { val=$1*1000/$2; printf "%.3f\n", val; } else print "unknown"; }') + if [ "$format" = csv ]; then + printf "%s,%s,%f,%f,%f,%f,%f,%f," $testdir "$build" $minperf $aveperf $maxperf $current $baselinePower $power + printf "%s\n" $perfPerWatt + else + printf "%-30s %-8s %12.2f %12.2f %12.2f %12.2f %12.2f %12.2f " $testdir "$build" $minperf $aveperf $maxperf $current $baselinePower $power + printf "%12s\n" $perfPerWatt + fi +} + +function calcBaselinePower { + workload=$1 + defaultPowerFile="idle-display-power.out" + powerFile=$defaultPowerFile + case $workload in + (shadowgrid2|suntemple|recentfling) + powerFile="idle-airplane-display-power.out" + if [ ! -f $powerFile ]; then + powerFile=$defaultPowerFile + fi;; + esac + if [ -f $powerFile ]; then + $POWERAVE 5 4.3 $powerFile + fi +} + +for t in $(cat tests) +do + echo .======================= $t ================================ + printHeader $t + for i in $testResults + do + cd $i + baseline="$(calcBaselinePower $t)" + if [ "$baseline" != "" ]; then + calcPerfData $i $t $baseline + else + echo "$i : no baseline current" + fi + cd - > /dev/null + done +done diff --git a/tests/workloads/pwrtest.sh b/tests/workloads/pwrtest.sh new file mode 100755 index 00000000..ca0f78dd --- /dev/null +++ b/tests/workloads/pwrtest.sh @@ -0,0 +1,362 @@ +# Script to gather perf and perf/watt data for several workloads +# +# Setup: +# +# - device connected to monsoon with USB passthrough enabled +# - network enabled (baseline will be measured and subtracted +# from results) (network needed for chrome, youtube tests) +# - the device is rebooted after each test (can be inhibited +# with "-r 0") +# +# Default behavior is to run each of the known workloads for +# 30 minutes gathering both performance and power data. +# +# The default time can be overridden with the -t option. To +# change individual test times, a config file can be specifed +# via -f with times for individual tests. Example file contents: +# +# idleTime=60 +# recentflingTime=60 +# chromeTime=60 +# youtubeTime=0 +# sysappsTime=60 +# suntempleTime=5 +# +# Output goes to the current directory. +# +# Examples: +# +# - Run all tests for 15 minutes (default is 30): ./pwrtest.sh -t 15 -R MDA20 +# +# - Use a config file for test times: ./pwrtest.sh -f ./myconfig -R MDA20 +# +# - Use a init file to setup device tuneables after each restart (this is +# a bash script which should include adb commands to set up device): +# ./pwrtest.sh -F devtunables +# +defaultTime=30 +garbageminutes=8 + +function Usage { + echo "Usage: $0 [OPTIONS]" + echo "-d device : device type (shamu, bullhead, ...)" + echo "-f configFile : config file to override individual test times" + echo "-g garbageMinutes : time to skip power measurement at beginning of test" + echo " default=$garbagetime minutes" + echo "-r restart : 0=no reboot between tests, 1=reboot (default)" + echo "-t defaultTimeMin : default time to run each test" + echo " default=$defaultTime minutes" + echo "-D cmddir : directory to find defs.sh" + echo "-F restartHookFile : file of commands to set device tunables after restart (optional)" + echo "-R release : release running on device (MDA20, 2054728, etc)" +} + +restart=1 +hz=5 +shadowgrid2TimeMax=25 + +CMDDIR=$(dirname $0 2>/dev/null) +CMDDIR=${CMDDIR:=.} + +MONSOON=monsoon.par + +while [ $# -gt 0 ] +do + case "$1" in + (-D) CMDDIR=$2; shift;; + (-r) restart=$2; shift;; + (-t) defaultTime=$2; shift;; + (-F) restartfile=$2; shift;; + (-g) garbageminutes=$2; shift;; + (-f) + configFile=$2; + echo "Reading configs from $configFile..." + . ./$configFile + shift;; + (-R) echo $2 > ./build; shift;; + (--) ;; + (--help) + Usage + exit 0;; + (*) + echo "Unknown option: $1" + Usage + exit 1;; + esac + shift +done + +. $CMDDIR/defs.sh -- + +devdir="/data/local/tmp" +suntempledir=${CMDDIR}/suntemple + +case $DEVICE in +(shamu|hammerhead) + HWUITEST=hwuitest + onSwipe="700 1847 700 400 50" + ;; +(*) + HWUITEST=hwuitest64 + onSwipe="500 1200 500 550 150" + ;; +esac + +scripts="defs.sh systemapps.sh recentfling.sh youtube.sh chromefling.sh $HWUITEST" + +if ! $MONSOON >/dev/null 2>&1; then + echo $MONSOON must be in your PATH >&2 + exit 1 +fi + +function usbpassthru { + if [ "$1" = off ]; then + state=off + else + state=on + fi + echo Setting usb pass-thru to $state + monsoon.par --usbpassthrough=$state +} + +function pwrcollect { + collectmin=$1 + collectmin=${collectmin:=60} + # samples = hz * 60 * minutes + ((samples=5*60*collectmin)) + monsoon.par --timestamp --samples $samples --hz 5 +} + +function copy_files { + adb shell mkdir -p $devdir + for file in $scripts + do + adb push $CMDDIR/$file $devdir + done +} + +function install_suntemple { + echo Checking for suntemple installation... + #stdest=/storage/sdcard0/obb/com.BrueComputing.SunTemple + stdest=/storage/emulated/0/obb/com.BrueComputing.SunTemple + dircontents=$(adb ls $stdest 2>/dev/null) + if [ "$dircontents" = "" ]; then + echo Installing suntemple... + adb install $suntempledir/*.apk + adb shell mkdir -p $stdest + adb push $suntempledir/main*obb $stdest + else + echo dircontents=$dircontents + echo Suntemple already installed. + fi +} + +function run_test { + testName=$1 + collectMinutes=$2 + collectOutput=${testName}-power-raw.out + powerOutput=${testName}-power.out + echo ----------------------------------------------------- + echo TEST: $testName + echo enabled Cores $(adb shell "cat /sys/devices/system/cpu/online") + date + echo ----------------------------------------------------- + usbpassthru off + pwrcollect $collectMinutes > $collectOutput 2>/dev/null + # take off the first 2min of samples + totalSamples=$(cat $collectOutput | wc -l) + # we throw away the first "garbageminutes" of the data + # since it is volatile + ((garbage=hz*60*garbageminutes)) + ((remaining=totalSamples-garbage)) + if [ $remaining -gt 0 ]; then + tail -$remaining $collectOutput > $powerOutput + else + cp $collectOutput $powerOutput + fi + echo power data for $testName copied to $collectOutput + usbpassthru on + sleep 10 + adb devices + sleep 10 +} + +function start_job { + cmdline="$1" + echo Running $cmdline + (adb shell "cd $devdir && nohup $cmdline > test.out") & + sleep 5 + kill %1 2>/dev/null +} + +function cleanup_job { + testName=$1 + processName=$2 + processName=${processName:=" sh "} + set -- $(adb shell ps | tr "\r" " " | grep "$processName") + echo killing PID=$2... + adb shell kill $2 + sleep 1 + echo copying test output to $testName... + adb pull $devdir/test.out + mv test.out ${testName}.out + if [ $restart -gt 0 ]; then + restart_device + else + doKeyevent HOME + fi +} + +function airplane_mode { + if [ "$1" = "on" ]; then + mode=true + setting=1 + else + mode=false + setting=0 + fi + adb shell settings put global airplane_mode_on $setting + adb shell am broadcast -a android.intent.action.AIRPLANE_MODE --ez state $mode + echo Set airplane mode to $mode +} + +function restart_device { + adb reboot + echo Wait 60s for device to restart... + sleep 60 + while ! adb root + do + echo Waiting for device to come up... + sleep 10 + done + echo Wait 30s to complete boot activities... + sleep 30 + echo Restart complete. + doSwipe $onSwipe + restartfile=${restartfile:="./restarthook"} + if [ -f $restartfile ]; then + # hook to change tunables after a restart + . $restartfile + fi +} + +usbpassthru on +adb devices 2>/dev/null + +airplane_mode off +if [ $restart -gt 0 ]; then + restart_device +fi + +echo Copying $scripts to device $devdir... +copy_files +tests="" + +# measure background power +idleTime=${idleTime:=$defaultTime} +if [ $idleTime -gt 0 ]; then + echo Test 1 : measure idle power for $idleTime minutes + run_test idle $idleTime + airplane_mode on + echo Restarting for power baseline in airplane mode... + restart_device + run_test idle-airplane $idleTime + airplane_mode off + # the screen blanks after 30 minutes. The first 2 minutes of the test + # have already been filtered off. For our power baseline, keep the first + # 20 minutes of the results + ((twentyminutes=hz*20*60)) + powerOutput="idle-power.out" + displayPowerOutput="idle-display-power.out" + airplanePowerOutput="idle-airplane-power.out" + airplaneDisplayPowerOutput="idle-airplane-display-power.out" + totalSamples=$(cat $powerOutput | wc -l) + if [ $twentyminutes -lt $totalSamples ]; then + head -$twentyminutes $powerOutput > $displayPowerOutput + head -$twentyminutes $airplanePowerOutput > $airplaneDisplayPowerOutput + else + cp $powerOutput $displayPowerOutput + cp $airplanePowerOutput $airplaneDisplayPowerOutput + fi + tests="$tests idle" +fi + +recentflingTime=${recentflingTime:=$defaultTime} +if [ $recentflingTime -gt 0 ]; then + echo $(date) Test 2 : recents fling for $recentflingTime minutes + airplane_mode on + adb shell "cd $devdir && ./systemapps.sh -A -T -i 1" + start_job "./recentfling.sh -N -i 1000 -d $DEVICE" + run_test recentfling $recentflingTime + cleanup_job recentfling + airplane_mode off + date + tests="$tests recentfling" +fi + +suntempleTime=${suntempleTime:=$defaultTime} +if [ $suntempleTime -gt 0 ]; then + echo $(date) Test 2 : run Sun Temple $suntempleTime minutes + airplane_mode on + install_suntemple + adb shell "am start $suntempleActivity" + run_test suntemple $suntempleTime + adb pull /sdcard/SunTemple/SunTemple/Saved/Logs/SunTemple.log + cleanup_job suntemple BrueComp + airplane_mode off + mv SunTemple.log suntemple.out + # grab the suntemple log + date + tests="$tests suntemple" +fi + +chromeTime=${chromeTime:=$defaultTime} +if [ $chromeTime -gt 0 ]; then + echo $(date) Test 3 : chrome fling for $chromeTime minutes + start_job "./chromefling.sh -i 1000 -d $DEVICE" + run_test chrome $chromeTime + cleanup_job chrome + date + tests="$tests chrome" +fi + +shadowgrid2Time=${shadowgrid2Time:=$defaultTime} +if [ $shadowgrid2Time -gt $shadowgrid2TimeMax ]; then + # we cap shadowgrid2 time since the display goes + # off after 30 minutes + $shadowgrid2Time=$shadowgrid2TimeMax +fi +if [ $shadowgrid2Time -gt 0 ]; then + airplane_mode on + echo $(date) Test 4 : shadowgrid2 for $shadowgrid2Time minutes + start_job "./$HWUITEST shadowgrid2 100000" + run_test shadowgrid2 $shadowgrid2Time + cleanup_job shadowgrid2 $HWUITEST + airplane_mode off + date + tests="$tests shadowgrid2" +fi + +youtubeTime=${youtubeTime:=$defaultTime} +if [ $youtubeTime -gt 0 ]; then + echo $(date) Test 5 : youtube for $youtubeTime minutes + start_job "./youtube.sh -i 1000 -d $DEVICE" + run_test youtube $youtubeTime + cleanup_job youtube + date + tests="$tests youtube" +fi + +sysappsTime=${sysappsTime:=$defaultTime} +if [ $sysappsTime -gt 0 ]; then + echo $(date) Test 6 : app switching for $sysappsTime minutes + start_job "./systemapps.sh -T -i 1000 -d $DEVICE" + run_test sysapps $sysappsTime + cleanup_job sysapps + date + tests="$tests sysapps" +fi + +echo Ran tests: $tests +echo $tests > tests + diff --git a/tests/workloads/recentfling.sh b/tests/workloads/recentfling.sh index 092c8d92..d495934e 100755 --- a/tests/workloads/recentfling.sh +++ b/tests/workloads/recentfling.sh @@ -44,6 +44,12 @@ case $DEVICE in upCount=6 UP="70 400 70 100 $flingtime" DOWN="70 100 70 400 $flingtime";; +(angler) + flingtime=150 + downCount=4 + upCount=3 + UP="500 1200 500 550 $flingtime" + DOWN="500 550 500 1200 $flingtime";; (bullhead) flingtime=200 downCount=5 @@ -122,7 +128,6 @@ do latency99=$5 if [ ${totalDiff:=0} -eq 0 ]; then echo Error: could not read frame info with \"dumpsys gfxinfo\" - exit 1 fi ((frameSum=frameSum+totalDiff)) @@ -132,7 +137,6 @@ do ((latency99Sum=latency99Sum+latency99)) if [ "$totalDiff" -eq 0 ]; then echo Error: no frames detected. Is the display off? - exit 1 fi ((jankPct=jankyDiff*100/totalDiff)) resetJankyFrames diff --git a/tests/workloads/systemapps.sh b/tests/workloads/systemapps.sh index a263e7d2..6c0db35c 100755 --- a/tests/workloads/systemapps.sh +++ b/tests/workloads/systemapps.sh @@ -23,12 +23,13 @@ # Other options are described below. # iterations=1 -tracecategories="gfx view am input memreclaim" +tracecategories="gfx am memreclaim" totaltimetest=0 forcecoldstart=0 waitTime=3.0 +memstats=0 -appList="gmail hangouts chrome youtube play home" +appList="gmail maps chrome youtube play home" function processLocalOption { ret=0 @@ -38,6 +39,7 @@ function processLocalOption { (-L) appList=$2; shift; ret=1;; (-T) totaltimetest=1;; (-W) waitTime=$2; shift; ret=1;; + (-M) memstats=1;; (*) echo "$0: unrecognized option: $1" echo; echo "Usage: $0 [options]" @@ -141,6 +143,7 @@ do if [ $iterations -gt 1 ]; then echo ========================================= echo Iteration $cur of $iterations + date echo ========================================= fi if [ $iterations -gt 1 -o $cur -eq 1 ]; then @@ -160,8 +163,11 @@ do if [ $totaltimetest -eq 0 ]; then tmpTraceOut="$tmpTraceOutBase-$app.out" >$tmpTraceOut - startInstramentation + startInstramentation "$app-$cur" else + if [ "$memstats" -gt 0 ]; then + startInstramentation "$app-$cur" 0 + fi if [ $appnum -eq 0 ]; then printf "%-8s %5s(ms) %3s(ms) %s %s\n" App Start Iter Jank Latency fi @@ -239,6 +245,8 @@ if [ $totaltimetest -gt 0 ]; then printf "%-10s %5.0f %5.0f\n" TOTAL $totaltime $diffTime fi +overallSum=0 +appCount=0 if [ $iterations -gt 1 -a $totaltimetest -eq 0 ]; then echo echo ========================================= @@ -258,7 +266,14 @@ if [ $iterations -gt 1 -a $totaltimetest -eq 0 ]; then ((ave90=l90/iterations)) ((ave95=l95/iterations)) ((ave99=l99/iterations)) - ((jankPct=100*janks/frames)) + if [ $frames -gt 0 ]; then + ((jankPct=100*janks/frames)) + fi printf "%-12s %5d %5d %5d %5d %5d %5d(%d%%) %d/%d/%d\n" $app $1 $ave $2 $4 $5 $janks $jankPct $ave90 $ave95 $ave99 + ((overallSum=overallSum+ave)) + ((appCount=appCount+1)) done + if [ $appCount -gt 0 ]; then + printf "Average Start Time: %.2f\n", $(echo $overallSum $appCount | awk '{ printf "%.2f\n", $1/$2 }') + fi fi diff --git a/tests/workloads/youtube.sh b/tests/workloads/youtube.sh new file mode 100755 index 00000000..2d6b1365 --- /dev/null +++ b/tests/workloads/youtube.sh @@ -0,0 +1,139 @@ +# +# Script to play a john oliver youtube video N times. +# For each iteration, Total frames and janky frames are reported. +# +# Options are described below. +# +iterations=10 +app=youtube +searchText="last week tonight with john oliver: online harassment" + +function processLocalOption { + ret=0 + case "$1" in + (-S) searchText="$2"; shift;; + (*) + echo "$0: unrecognized option: $1" + echo; echo "Usage: $0 [options]" + echo "-i iterations" + echo "-S youtube search text" + echo "-d device" + exit 1;; + esac + return $ret +} + +CMDDIR=$(dirname $0 2>/dev/null) +CMDDIR=${CMDDIR:=.} +. $CMDDIR/defs.sh + +case $DEVICE in +(angler) + searchButton="860 177" + selectFirstVideo="225 400" + enableControls="1000 610" + fullScreen="1011 632" + ;; +(shamu) + searchButton="1200 160" + selectFirstVideo="480 653" + enableControls="1377 812" + fullScreen="1377 812" + ;; +(bullhead|hammerhead) + searchButton="860 177" + selectFirstVideo="225 400" + enableControls="1000 610" + fullScreen="1011 632" + ;; +(volantis) + searchButton="1356 93" + selectFirstVideo="378 264" + enableControls="1464 812" + fullScreen="1480 835" + ;; +(*) + echo "Error: No display information available for $DEVICE" + exit 1;; +esac + +function swipe { + count=0 + while [ $count -lt $2 ] + do + echo doSwipe... + doSwipe $1 + ((count=count+1)) + done + sleep 1 +} + +cur=1 +frameSum=0 +jankSum=0 +latency90Sum=0 +latency95Sum=0 +latency99Sum=0 + +doKeyevent HOME +sleep 0.5 +resetJankyFrames $(getPackageName $app) + +while [ $cur -le $iterations ] +do + t=$(startActivity $app) + sleep 4.0 + doTap $searchButton + sleep 1.0 + doText "$searchText" + sleep 1.0 + doKeyevent ENTER + sleep 5.0 + doTap $selectFirstVideo + sleep 10.0 + doTap $fullScreen + sleep 0.5 + doTap $fullScreen + # 15 minutes + ((vidTime=60*15)) + sleep $vidTime + doKeyevent BACK + sleep 0.5 + doKeyevent BACK + sleep 0.5 + doKeyevent BACK + sleep 0.5 + + set -- $(getJankyFrames $(getPackageName $app)) + totalDiff=$1 + jankyDiff=$2 + latency90=$3 + latency95=$4 + latency99=$5 + if [ ${totalDiff:=0} -eq 0 ]; then + echo Error: could not read frame info with \"dumpsys gfxinfo\" + fi + + ((frameSum=frameSum+totalDiff)) + ((jankSum=jankSum+jankyDiff)) + ((latency90Sum=latency90Sum+latency90)) + ((latency95Sum=latency95Sum+latency95)) + ((latency99Sum=latency99Sum+latency99)) + if [ "$totalDiff" -eq 0 ]; then + echo Error: no frames detected. Is the display off? + fi + ((jankPct=jankyDiff*100/totalDiff)) + resetJankyFrames $(getPackageName $app) + + + echo Frames: $totalDiff latency: $latency90/$latency95/$latency99 Janks: $jankyDiff\(${jankPct}%\) + ((cur=cur+1)) +done +doKeyevent HOME +((aveJankPct=jankSum*100/frameSum)) +((aveJanks=jankSum/iterations)) +((aveFrames=frameSum/iterations)) +((aveLatency90=latency90Sum/iterations)) +((aveLatency95=latency95Sum/iterations)) +((aveLatency99=latency99Sum/iterations)) +echo AVE: Frames: $aveFrames latency: $aveLatency90/$aveLatency95/$aveLatency99 Janks: $aveJanks\(${aveJankPct}%\) |