path: root/benchmarks/benchmarks_run_target.sh

#!/bin/bash
#
# Copyright (c) 2016-2021, Linaro Ltd.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

readonly local_path=$(dirname "$0")
source "${local_path}/../utils/utils.sh"
source "${local_path}/../utils/utils_test.sh"
source "${local_path}/../utils/utils_android.sh"
source "${local_path}/../utils/utils_android_root.sh"
source "${local_path}/../utils/utils_benchmarks.sh"
source "${local_path}/../devices/cpu_freq_utils.sh"

readonly timer_name="Target Benchmarks"
readonly LOG_DIRECTORY="$(get_workspace)"

readonly default_iterations=10

target_cfg_file=""


# shellcheck disable=SC2034
declare -A options_format=(
  ["help"]="p:usage()"
  ["h"]="r:&help"
  ["verbose"]="p:enable_verbose()"
  ["v"]="r:&verbose"
  ["mode"]="all"
  ["linux"]="false"
  ["sudo"]="false"
  ["x86"]="false"
  ["cpu"]="all"
  ["skip-build"]="false"
  ["skip-run"]="false"
  ["list-devices"]="p:list_devices()"
  ["iterations"]="${default_iterations}"
  ["dump-cfg"]="false"
  ["dump-cfg-methods"]="benchmarks"
  ["target-device"]=""
  ["isa-features"]="default"
)
declare -A options=()
declare -a benchmarks=()

validate_options() {
  local -r mode="${options["mode"]}"
  validate_mode_option "${mode}"

  local -r cpu="${options["cpu"]}"
  if ! [[ ${cpu} == "all" || ${cpu} == "default" ]]; then
    validate_cluster "${cpu}"
  fi

  local -r iterations="${options["iterations"]}"
  if [[ ! ${iterations} =~ ^[0-9]+$ ]]; then
    log E "Invalid number of iterations: ${iterations}"
    exit 1
  fi
  local -r skip_build="${options["skip-build"]}"
  local -r skip_run="${options["skip-run"]}"
  if [[ "${skip_build}" == "true" && "${skip_run}" == "true" ]]; then
    log E "skip_run and skip_build can't true in the same execution. Please set the right option"
    exit 1
  fi

  # "target-device" should be only used for running on arm targets.
  local -r x86_option="${options["x86"]}"
  local -r target_device_option="${options["target-device"]}"
  if [[ "${x86_option}" == "true" && "${target_device_option}" != "" ]]; then
    log E "Using X86 and specifying Arm target device can't be used together"
    exit 1
  fi
}

validate_benchmarks_names() {
  local -r workspace="$(get_workspace)"
  local benchmark
  for benchmark in "${benchmarks[@]}"; do
    local benchmark_java_file="${workspace}/benchmarks/${benchmark}.java"
    if [[ ! -f "${benchmark_java_file}" ]]; then
      log E "Java file ${benchmark_java_file} does not exist for benchmark ${benchmark}"
      exit 1
    fi
  done
}

usage() {
  log I "Usage: $0 [OPTION]... [BENCHMARK]..."
  log I ""
  log I "This script should be used for running BENCHMARK(s) on a target device."
  log I "The script expects a device to be connected. Use --list-devices to show supported"
  log I "devices."
  log I "With no BENCHMARK, all available benchmarks are run."
  log I ""
  log I "-------------------------------------------"
  log I " -h, --help                - help"
  log I " -v, --verbose             - verbose"
  log I " --list-devices            - List the devices supported by this script."
  log I " --mode <all|32|64>        - Run benchmarks for the specified mode(s)."
  log I "                             (default: all)"
  log I " --cpu <all|cluster_name|default> - CPU mode."
  log I "                             \"cluster_name\": Run with only cores specified by"
  log I "                             \`cluster_name\` and pin their frequency."
  log I "                             \"all\": Run one cluster of cores at a time until all"
  log I "                             clusters/cores have been run. Each cluster is run pinned to"
  log I "                             a specific frequency to ensure consistency between tests."
  log I "                             \"default\": Run with unaltered default CPU configuration"
  log I "                             (no pinning)."
  log I "                             (default: all)"
  log I " --iterations <n>          - The number of iterations to run the benchmarks for."
  log I "                             (default: $default_iterations)"
  log I " --target-device <device>  - Use specific lunch target and configuration for Arm target"
  log I "                             platform"
  log I "                             (default: \"\" - which will cause a default target arch"
  log I "                             to be used."
  log I " --linux                   - Compile ART for a non-android kernel"
  log I " --x86                     - Compile ART for x86 architecture (otherwise, arm is implied)"
  log I " --sudo                    - Use \`sudo\` for \`adb shell\` commands in targets without"
  log I "                             support for \`adb root\`"
  log I " --skip-build <false|true> - Skips the build step and runs benchmark with prebuilt"
  log I "                             artifacts from \$OUT directory."
  log I "                             This option is mainly used for automation."
  log I "                             Make sure \`adb\` and \`dx\` are in your PATH when using"
  log I "                             this option."
  log I "                             (default: false)"
  log I " --skip-run <false|true>   - Skips the running benchmark but just builds the artifacts"
  log I "                             (default: false)"
  log I " --dump-cfg                - Dump control-flow graphs. Depending on the mode they will be"
  log I "                             either in bench.arm64.cfg or bench.arm32.cfg."
  log I " --dump-cfg-methods <all|benchmarks|method names>"
  log I "                           - Restrict dumped CFG data to methods."
  log I "                             \"all\": No restriction. Dump CFG of every compiled"
  log I "                             method. Note: it might be very big."
  log I "                             \"benchmarks\": Restrict to methods from benchmarks."
  log I "                             method names: a comma-separated list of names."
  log I "                             Dump CFGs of the methods which names contains"
  log I "                             one of the name from the list."
  log I "                             Example: --dump-cfg-methods Linpack.matgen,Sort.bubbleSort"
  log I "                             (default: benchmarks)"
  log I " --isa-features            - specify isa features to be used."
  log I "                             Possible values: default, runtime, a list of comma-separated"
  log I "                             feature names. When the list is used, '-' before a feature"
  log I "                             name means the ART compiler must not use the feature for"
  log I "                             code generation"
  log I "-------------------------------------------"
  log I "Default Configuration:"
  log I " --default                 - Default benchmark configuration, equivalent to"
  log I "                             \`--mode all --cpu all --iterations $default_iterations\`."
  log I "-------------------------------------------"
  exit 0
}

list_devices() {
  list_devices_from_config "${local_path}/../devices/config/*"
  exit 0
}

# Arguments:
#   ${1}: isa
create_target_cmdline_sh() {
  local isa="${1}"

  local dump_cfg=""
  if ${options["dump-cfg"]}; then
    target_cfg_file="/data/local/tmp/bench.${1}.cfg"
    dump_cfg="--dump-cfg=${target_cfg_file}"
    local verbose_methods="${options["dump-cfg-methods"]}"
    if [[ "${options["dump-cfg-methods"]}" == "benchmarks" ]]; then
      verbose_methods=""
      # Restrict methods to methods from benchmarks.
      # If there are user-specified benchmarks, restrict to them.
      local benchmark
      for benchmark in "${benchmarks[@]}"; do
        verbose_methods+=",$(basename "${benchmark}")"
      done
      # If there are no user-specified benchmarks, restrict to all benchmarks.
      # As all benchmarks are in the package 'benchmarks', specifying 'benchmarks' will restrict
      # methods to methods from benchmarks.
      if [[ ! -z "${verbose_methods}" ]]; then
        verbose_methods="${verbose_methods:1}"
      else
        verbose_methods="benchmarks"
      fi
    fi
    if [[ "${verbose_methods}" != "all" ]]; then
      verbose_methods="${verbose_methods:1}"
      dump_cfg+=" --verbose-methods=${verbose_methods}"
    fi
  fi

  local isa_features=${options["isa-features"]}

  read -r -d '' target_script << EOM
    $(get_target_art_test_env_vars_exports)
    cd /data/local/tmp

    rm -rf /data/local/tmp/oat/${isa}
    mkdir -p /data/local/tmp/oat/${isa}
    before=\$EPOCHREALTIME
    $(get_target_dex2oat_cmd \
        "${isa}" \
        "/data/local/tmp/bench.apk" \
        "${isa_features}") \
        $(get_target_art_test_bootclasspath_for_dex2oat) -j1 ${dump_cfg}
    after=\$EPOCHREALTIME

    echo "Before: "\$before
    echo "After: "\$after

    if [[ \$? -ne 0 ]]; then
      rm -rf /data/local/tmp/oat/${isa}
      exit 1
    fi

    $(get_target_art_test_dalvikvm_cmd \
        "/data/local/tmp" \
        "/data/local/tmp/bench.apk" \
        "${isa_features}") \$@
EOM

  safe adb_shell "mkdir -p ${ART_TEST_CHROOT}/data/local/tmp/"
  safe adb_shell "echo '${target_script}' > ${ART_TEST_CHROOT}/data/local/tmp/cmdline.sh"
}

copy_cfg_file_from_device() {
  [[ -n "${target_cfg_file}" ]]
  if_error "'target_cfg_file' is empty."
  safe adb pull "${ART_TEST_CHROOT}/${target_cfg_file}" \
    "${LOG_DIRECTORY}/$(basename "${target_cfg_file}")"
}

run_benchmarks() {
  local isa="arm"
  if [[ "$1" == "64" ]]; then
    isa+="64"
  fi

  create_target_cmdline_sh "${isa}"
  export ART_COMMAND="chroot ${ART_TEST_CHROOT} sh /data/local/tmp/cmdline.sh "

  local run_cmd_options=()
  run_cmd_options+=("--target")
  run_cmd_options+=("--iterations" "${options["iterations"]}")
  run_cmd_options+=("--mode" "$1")
  run_cmd_options+=("--output-json" "${LOG_DIRECTORY}/$2_$1_$3_aot.json")
  run_cmd_options+=("--target-copy-path" "${ART_TEST_CHROOT}/data/local/tmp")
  run_cmd_options+=("--output-oat" "${ART_TEST_CHROOT}/data/local/tmp/oat/${isa}/bench.odex")

  for benchmark in "${benchmarks[@]}"; do
    run_cmd_options+=("--filter" "${benchmark}")
  done

  start_adb_section "run_benchmarks_$2_$1_$3"
  ./benchmarks/run.py "${run_cmd_options[@]}"
  local -r return_code=$?
  end_adb_section "run_benchmarks_$2_$1_$3" "${return_code}"
  return "${return_code}"
}

# Arguments:
#   ${1} - CPU mode (bitness)
run_all_benchmarks() {
  local -r bitness="$1"
  local -r target_device=$(retrieve_target_product_name)
  local -r cpu="${options["cpu"]}"
  local -r path_to_devices="${local_path}/../devices"
  set_freq_and_run run_benchmarks "${bitness}" "${cpu}" "${target_device}" "${path_to_devices}"
}

main() {
  exit_on_failure arguments_parser options_format options benchmarks -- "$@"
  readonly options
  validate_options
  dump_options
  local -r mode="${options["mode"]}"
  local -r skip_build="${options["skip-build"]}"
  local -r skip_run="${options["skip-run"]}"

  if android_build_already_setup; then
    log E "This test does not support environment targets. Please re-run in a clean environment."
    exit 1
  fi

  start_test "${timer_name}"

  for bits in 32 64; do
    target_cfg_file=""
    if [[ "$mode" != "all" && "$mode" != "$bits" ]]; then
      log I "Skipping ${bits}bit benchmarks."
      continue
    fi
    log I "Starting ${bits}bit benchmarks."

    # Set environment variables.
    set_environment_for_benchmark_run options

    if [[ "${skip_build}" == "false" ]]; then
      # Build target.
      build_target "${bits}"
      if is_full_source_tree; then
        build_boot_oat "${bits}"
      fi
    else
      setup_android_target_from_bits "${bits}"
    fi

    if [[ "${skip_run}" == "true" ]]; then
      log I "Skipping running ${bits}bit benchmarks."
      continue
    fi

    export_javac_bootclasspath

    buildbot_device_prepare "${bits}"
    run_all_benchmarks "${bits}"
    if ${options["dump-cfg"]}; then
      copy_cfg_file_from_device
    fi
  done

  end_test "${timer_name}"
}

main "$@"