path: root/tests/test_art_target.sh

#!/bin/bash

# Target tests.
#
# Jenkins link:
# https://android-build.linaro.org/jenkins/job/linaro-art-tip-build-ARTTargetTest

readonly local_path=$(dirname "$0")
source "${local_path}/../utils/utils.sh"
source "${local_path}/../utils/utils_android.sh"

readonly target_timer_name="Target Test"
readonly log_directory="$(get_workspace)"
readonly test_time_file="${log_directory}/time_test.txt"

declare -A options
tests_summary=
return_code=0

init_options() {
  options["32bit"]="false"
  options["64bit"]="false"
  options["concurrent_gc"]="false"
  options["debug"]="false"
  options["gtests"]="false"
  options["interpreter"]="false"
  options["jdwp"]="false"
  options["jit"]="false"
  options["keep_failures"]="false"
  options["keep_going"]="false"
  options["libcore"]="false"
  options["optimizing"]="false"
  options["valgrind"]="false"
}

set_default_options() {
  options["32bit"]="true"
  options["64bit"]="true"
  options["concurrent_gc"]="false"
  options["debug"]="false"
  options["gtests"]="true"
  options["interpreter"]="false"
  options["jdwp"]="false"
  options["jit"]="false"
  options["keep_going"]="true"
  options["libcore"]="false"
  options["optimizing"]="true"
  options["valgrind"]="false"
}

validate_options() {
  # Check that either 32bit or 64bit or both have been selected.
  if ! ${options["32bit"]} && ! ${options["64bit"]}; then
    log E "Please select at least one \"Test Mode\" or use --default."
    log E "See the options ( -h | --help ) for more info."
    exit 1
  fi

  # Check that at least one test has been selected.
  if ! ${options["gtests"]} && ! ${options["optimizing"]} \
      && ! ${options["interpreter"]} && ! ${options["jit"]} \
      && ! ${options["libcore"]} && ! ${options["jdwp"]}; then
    log E "Please select at least one of the \"Tests\" to be run or use --default."
    log E "See the options ( -h | --help ) for more info."
    exit 1
  fi

  # Check that if running with Valgrind has been requested,
  # at least one of the supported tests has been selected.
  if ! ${options["gtests"]} && ${options["valgrind"]}; then
    log E "Running under Valgrind is supported only for the gtests at the moment. Please use --gtests."
    log E "See the options ( -h | --help ) for more info."
    exit 1
  fi
}


set_option() {
  local current_option=${1#"--"}
  # shellcheck disable=SC2144
  # shellcheck disable=SC2102
  if [[ -v options[${current_option}] ]]; then
    options[${current_option}]="true"
  else
    # There might be something wrong with the option parser since this option
    # does not have a default.
    abort
  fi
}

dump_options() {
  log I "Running tests with the following configuration:"
  for key in "${!options[@]}"; do
    log I "options[${key}] = ${options[${key}]}"
  done
}

usage() {
  log I "$0"
  log I "This script is used to run the Target Tests by Jenkins.\n"
  log I "Note that for now this test does not support environment targets."
  log I "This means that you should not run the script from an android env,"
  log I "where you have already set-up a lunch target.\n"
  log I "This script expects a 64bit device connected, that responds to adb.\n"
  log I " -h|--help       - help"
  log I " -v|--verbose    - verbose"
  log I "-------------------------------------------"
  log I "Test Modes:"
  log I " --32bit         - run 32bit tests."
  log I " --64bit         - run 64bit tests."
  log I "-------------------------------------------"
  log I "Test Options:"
  log I " --debug         - run tests in debug mode."
  log I " --concurrent_gc - run tests using the concurrent gc."
  log I " --keep_going    - don't stop at the first failing test."
  log I " --valgrind      - run under Valgrind (only for gtests at the moment)."
  log I " --keep_failures - keep failing tests around (useful for debugging)."
  log I "-------------------------------------------"
  log I "Tests:"
  log I " --gtests        - run gtest tests."
  log I " --optimizing    - run optimizing tests."
  log I " --interpreter   - run interpreter tests."
  log I " --jit           - run jit tests."
  log I " --libcore       - run libcore tests."
  log I " --jdwp          - run jdwp tests."
  log I "-------------------------------------------"
  log I "Default Configuration (also used by Jenkins):"
  log I " --default       - default test configuration, also used by jenkins."
  log I "-------------------------------------------"
}

# Note: We don't use getopts since it only deals with short options.
arguments_parser() {
  init_options
  while [[ $# -gt 0 ]]; do
    case "$1" in
      -v|--verbose)
        enable_verbose
        ;;
      -h|--help)
        usage
        exit 0
        ;;
      # Test Modes:
      --32bit|--64bit)
        set_option "$1"
        ;;
      # Test Options:
      --debug|--concurrent_gc|--keep_going|--valgrind|--keep_failures)
        set_option "$1"
        ;;
      # Tests:
      --gtests|--optimizing|--interpreter|--jit|--libcore|--jdwp)
        set_option "$1"
        ;;
      # Default:
      --default)
        set_default_options
        ;;
      *)
        log E "Invalid option: $1"
        exit 1
        ;;
    esac
    shift
  done
  readonly options
  validate_options
}

start_section() {
  log I "Starting section: $1"
  tests_summary+=$(printf "%-25s" "$1")
  start_timer "$1"
  start_logging "${log_directory}/log_build_$1.txt"
}

start_adb_section() {
  start_section "$1"
  adb_clear_logcat
}

end_section() {
  stop_logging "${log_directory}/log_build_$1.txt"
  stop_timer "$1"
  local test_time=$(print_timer "$1")
  tests_summary+=$(printf "TOOK: %-10s" "${test_time}")

  if [[ $2 -ne 0 ]]; then
    log E "Section: $1 FAILED"
    tests_summary+="FAILED\n"
    return_code=$2

    if ! ${options["keep_going"]}; then
      log E "--keep_going is not set. Stop on the first failed test."
      exit "$2"
    fi
  else
    log S "Section $1 PASSED"
    tests_summary+="PASSED\n"
  fi
}

end_adb_section() {
  end_section "$1" "$2"
  adb_dump_logcat "${log_directory}/log_logcat_$1.txt"
}

skip_section() {
  log I "Skipping section $1"
  stop_logging "${log_directory}/log_build_$1.txt"
  stop_timer "$1"
  tests_summary+=$(printf "TOOK: %-10s" "0s")
  tests_summary+="SKIPPED\n"
}

set_common_environment() {
  # All these should be in sync with:
  # https://chromium.googlesource.com/chromium/tools/build/+/master/scripts/slave/recipes/art.py
  log I "Setting-up the test environment"

  # ART Tests to be run.
  export ART_TEST_INTERPRETER=true
  export ART_TEST_JIT=true
  export ART_TEST_OPTIMIZING=true
  export ART_USE_OPTIMIZING_COMPILER=true
  # Should the script stop after the first failing test?
  export ART_TEST_KEEP_GOING=true
  # ART test env setting.
  export ART_BUILD_HOST_DEBUG=false
  export ART_TEST_ANDROID_ROOT="/data/local/tmp/system"
  export ART_TEST_FULL=false
  export ART_TEST_RUN_TEST_2ND_ARCH=false
  export ART_TEST_NO_SYNC=true
  # Java settings.
  export LEGACY_USE_JAVA7=true
  # Jack settings.
  export JACK_SERVER=false
  export JACK_REPOSITORY="${PWD}/prebuilts/sdk/tools/jacks"
  # Dex2oat settings.
  export USE_DEX2OAT_DEBUG=false

  if ! ${options["debug"]}; then
    export ART_TEST_RUN_TEST_DEBUG=false
    export ART_TEST_RUN_TEST_NDEBUG=true
  fi

  if ${options["concurrent_gc"]}; then
    export ART_USE_READ_BARRIER=false
    export ART_HEAP_POISONING=false
  fi

  if ${options["keep_failures"]}; then
    export ART_TEST_RUN_TEST_ALWAYS_CLEAN=false
    export ART_TEST_QUIET=false
  fi
}

build_target() {
  start_section "build_target_$1"

  if [[ $1 -eq 32 ]]; then
    setup_android_target "arm_krait-eng"
    export CUSTOM_TARGET_LINKER="${ART_TEST_ANDROID_ROOT}/bin/linker"
  elif [[ $1 -eq 64 ]]; then
    setup_android_target "armv8-eng"
    export CUSTOM_TARGET_LINKER="${ART_TEST_ANDROID_ROOT}/bin/linker64"
    export CUSTOM_TARGET_2ND_LINKER="${ART_TEST_ANDROID_ROOT}/bin/linker"
  else
    log E "Mode not specified correctly. Make sure your bits are complete ($@)."
    abort
  fi

  safe ./art/tools/buildbot-build.sh -j"${jcpu_count}" --target
  end_section "build_target_$1" "$?"
}

configure_cpus() {
  safe "${local_path}/../devices/set_cpu_freq.sh" --default
}

device_setup() {
  start_section "device_setup_$1"
  # Get adb root - needed otherwise device_cleanup() would fail.
  safe adb root
  # Set up buildbot device.
  safe ./art/tools/setup-buildbot-device.sh
  end_section "device_setup_$1" "$?"
}

device_cleanup() {
  start_section "device_cleanup_$1"
  # Clean up all vestiges of ART testing on device.
  safe adb shell rm -rf /data/local/tmp /data/art-test /data/nativetest
  end_section "device_cleanup_$1" "$?"
}

sync_target() {
  start_adb_section "sync_target_$1"
  safe make ART_TEST_NO_SYNC=false -j"${jcpu_count}" "test-art-target-sync"
  end_adb_section "sync_target_$1" "$?"
}


# Build (run) a test target using the Android build system.
# If a test failed, add it to the list of failed tests.
# Arguments:
#   ${1}: test target.
#   ${2}: -j count.
#   ${3}: test name.
#   ${4}: bitness.
run_test() {
  start_adb_section "$2_$3"

  # Check if we should skip section
  if ! ${options["$2"]}; then
    skip_section "$2_$3"
    return
  fi

  # Get the No. of CPUs.
  local target_cpu_count=$(safe adb_shell_strip grep -c "^processor" /proc/cpuinfo)

  # Android scripts don't play nice with nounset (error out on expansion of unset variables).
  disable_error_on_unset_expansion
  # Adding dist will filter out time sensitive tests such as 055-enum-performance.
  make "-j${target_cpu_count}" "$1" "dist"
  end_adb_section "$2_$3" "$?"
  enable_error_on_unset_expansion
}

test_gtest() {
  local test_target=""

  if ${options["valgrind"]}; then
    test_target="valgrind-"
  fi

  test_target+="test-art-target-gtest$1"
  run_test "${test_target}" "gtests" "$1"
}

test_optimizing() {
  ART_TEST_RUN_TEST_DEBUGGABLE=true \
    run_test "test-art-target-run-test-optimizing" "optimizing" "$1"
}

test_interpreter() {
  run_test "test-art-target-run-test-interpreter" "interpreter" "$1"
}

test_jit() {
  run_test "test-art-target-run-test-jit" "jit" "$1"
}

test_libcore() {
  start_adb_section "libcore_$1"

  if ! ${options["libcore"]}; then
    skip_section "libcore_$1"
    return
  fi

  local test_options="--mode=device --variant=X$1"
  if ${options["debug"]}; then
    test_options+=" --debug"
  fi

  safe ./art/tools/run-libcore-tests.sh "${test_options}"
  end_adb_section "libcore_$1" "$?"
}


test_jdwp() {
  start_adb_section "jdwp_$1"

  if ! ${options["jdwp"]}; then
    skip_section "jdwp_$1"
    return
  fi

  local test_options="--mode=device --variant=X$1"
  if ${options["debug"]}; then
    test_options+=" --debug"
  fi

  safe ./art/tools/run-jdwp-tests.sh "${test_options}"
  end_adb_section "jdwp_$1" "$?"
}

main() {
  arguments_parser "$@"
  dump_options

  if android_build_already_setup; then
    log E "This test does not support environment targets. Please re-run in a clean environment."
    exit 1
  fi

  start_timer "${target_timer_name}"
  for bits in 32 64; do
    if ! ${options["${bits}bit"]}; then
      log I "Skipping ${bits}bit tests."
      continue
    fi
    log I "Starting ${bits}bit tests."

    # Set environemnt variables.
    set_common_environment

    # Build target.
    build_target "${bits}"

    configure_cpus

    # Setup & sync device.
    device_setup "${bits}"
    device_cleanup "${bits}"
    sync_target "${bits}"

    # Start testing.
    test_gtest "${bits}"
    test_optimizing "${bits}"
    test_interpreter "${bits}"
    test_jit "${bits}"
    test_libcore "${bits}"
    test_jdwp "${bits}"
  done
  stop_timer "${target_timer_name}"

  log I "$0 Finished!"
  dump_timer "${target_timer_name}" "${test_time_file}"

  local log_level="S"
  if [[ ${return_code} -ne 0 ]]; then
    log_level="E"
  fi
  log "${log_level}" "Test Summary:\n${tests_summary}"

  exit "${return_code}"
}

main "$@"