path: root/tests/test_art_target.sh

#!/bin/bash
#
# Copyright (c) 2016-2021, Linaro Ltd.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Target tests.
#
# Jenkins link:
# https://android-build.linaro.org/jenkins/job/linaro-art-tip-build-ARTTargetTest

readonly local_path=$(dirname "$0")
source "${local_path}/../utils/utils.sh"
source "${local_path}/../utils/utils_test.sh"
source "${local_path}/../utils/utils_android.sh"
source "${local_path}/../utils/utils_android_root.sh"
source "${local_path}/../utils/utils_run.sh"

readonly target_timer_name="Target Test"
readonly LOG_DIRECTORY="$(get_workspace)"

job_count=6

# shellcheck disable=SC2034
declare -A options_format=(
  ["help"]="p:usage()"
  ["h"]="r:&help"
  ["verbose"]="p:enable_verbose()"
  ["v"]="r:&verbose"
  ["default"]="p:set_defaults_wrapper()"
  ["32bit"]="false"
  ["64bit"]="false"
  ["concurrent-gc"]="false"
  ["debug"]="false"
  ["gtest"]="false"
  ["interpreter"]="false"
  ["jdwp"]="false"
  ["jit"]="false"
  ["keep-failures"]="false"
  ["keep-going"]="false"
  ["libcore"]="false"
  ["optimizing"]="false"
  ["skip-build"]="false"
  ["single-test"]=""
  ["jobs"]=""
  ["rerun-failed-tests"]="false"
  ["restart-adb-server"]="false"
  ["gcstress"]="false"
  ["fvp"]="false"
  ["isa-features"]=""
  ["dump-cfg"]=""
  ["gdb"]="false"
  ["gdb-dex2oat"]="false"
  ["gdb-dex2oat-args"]=""
)
declare -A options=()

set_defaults_wrapper() {
  new_opts_string=$(set_default_options "$(declare -p options)" "target" "${job_count}")
  declare -A -g options=${new_opts_string#*=}
}

# Return true if adb server is run on a different machine than
# test_art_target.sh.
is_remote_adb_server() {
  # In CI an adb server is run a remote machine different from the machine
  # used to run test_art_target.sh. In this case SERVER_IPADDRESS is set.
  [[ -v SERVER_IPADDRESS ]]
}

validate_options() {
  # Check that at least one test has been selected.
  if ! ${options["gtest"]} && ! ${options["optimizing"]} \
      && ! ${options["interpreter"]} && ! ${options["jit"]} \
      && ! ${options["libcore"]} && ! ${options["jdwp"]} \
      && [[ -z ${options["single-test"]} ]]; then
    log E "Please select at least one of the \"Tests\" to be run or use --default."
    log E "See the options ( -h | --help ) for more info."
    exit 1
  fi

  # Check for conflict between bitness in test name and bitness option
  if [[ "${options["single-test"]}" = *"32" ]] && ${options["64bit"]}; then
    log E "The test name specifies to run the test in the 32-bit mode but"
    log E "'--64bit' is used. The test mode option can be omitted in this case."
    exit 1
  fi

  if [[ "${options["single-test"]}" = *"64" ]] && ${options["32bit"]}; then
    log E "The test name specifies to run the test in the 64-bit mode but"
    log E "'--32bit' is used. The test mode option can be omitted in this case."
    exit 1
  fi

  if [[ -n "${options["jobs"]}" ]] && [[ ! ${options["jobs"]} =~ ^[0-9]+$ ]]; then
    log E "The --jobs option must be followed by an integer."
    exit 1
  fi

  if ${options["restart-adb-server"]} && is_remote_adb_server; then
    log E "'--restart-adb-server' can not be used to restart a remote adb server."
    exit 1
  fi

  if ${options["gcstress"]} && ! ${options["optimizing"]} && ! ${options["interpreter"]} \
      && ! ${options["jit"]} && { [[ -z ${options["single-test"]} ]] \
      || is_single_test_gtest "${options["single-test"]}"; }; then
    log E "The --gcstress option must be used only for run-tests: optimizing, jit or interpreter."
    exit 1
  fi

  if ${options["fvp"]}; then
    if [[ -n "${options["jobs"]}" ]]; then
      log E "The --jobs options must not be used with --fvp; it resets the number of jobs \
            to 1."
      exit 1
    fi
    job_count="1"
  fi

  if [[ -n "${options["dump-cfg"]}" ]] && [[ -z "${options["single-test"]}" ]]; then
    log E "Can only dump .cfg for a single test."
    exit 1
  fi

  if ${options["gdb"]} || ${options["gdb-dex2oat"]}; then
    log E "Debugging is not supported for target tests."
    exit 1
  fi
}

usage() {
  log I "$0"
  log I "This script is used to run the Target Tests by Jenkins.\n"
  log I "Note that for now this test does not support environment targets."
  log I "This means that you should not run the script from an android env,"
  log I "where you have already set-up a lunch target.\n"
  log I "This script expects a 64bit device connected, that responds to adb.\n"
  log I "Command-line options:"
  log I " -h|--help       - help"
  log I " -v|--verbose    - verbose"
  log I "-------------------------------------------"
  log I "Test Modes:"
  log I " --32bit         - run 32bit tests."
  log I " --64bit         - run 64bit tests."
  log I " Not providing a test mode means to run both 32bit and 64bit tests."
  log I "-------------------------------------------"
  log I "Test Options:"
  log I " --debug                 - run tests in debug mode."
  log I " --concurrent-gc         - run tests using the concurrent gc."
  log I " --jobs                  - a number of jobs on a target."
  log I "                           The default value is 3."
  log I " --keep-going            - don't stop at the first failing test."
  log I " --keep-failures         - keep failing tests around (useful for debugging)."
  log I " --skip-build            - skip building ART and its dependencies."
  log I " --rerun-failed-tests    - rerun failed tests."
  log I " --gcstress              - use gc stress testing for run-tests."
  log I " --fvp                   - configure and run tests on Arm FVP (sets"
  log I "                           --jobs to 1)."
  log I " --isa-features          - specify isa features to be used."
  log I "                           Possible values: default, runtime, a list of comma-separated"
  log I "                           feature names. When the list is used, '-' before a feature name"
  log I "                           means the ART compiler must not use the feature for code"
  log I "                           generation."
  log I " --dump-cfg <path>       - dump .cfg to the specified host full path. Only runs for a"
  log I "                           single test"
  log I "-------------------------------------------"
  log I "Tests:"
  log I " --gtest              - run gtest tests."
  log I " --optimizing         - run optimizing tests."
  log I " --interpreter        - run interpreter tests."
  log I " --jit                - run jit tests."
  log I " --libcore            - run libcore tests."
  log I " --jdwp               - run jdwp tests."
  log I " --single-test <test> - run specified test only"
  log I "-------------------------------------------"
  log I "Default Configuration (also used by Jenkins):"
  log I " --default       - default test configuration, also used by jenkins."
  log I "-------------------------------------------"
  log I "Misc:"
  log I " --restart-adb-server - restart a local adb server."
  log I "                        The adb from prebuilts/runtime is used."
  log I "-------------------------------------------"
  exit 0
}

# Sources envsetup.sh file.
source_test_environment() {
  source_android_environment_default

  if ${options["fvp"]}; then
    export ART_TEST_RUN_ON_ARM_FVP=true
  fi
}

set_environment() {
  source_test_environment

  set_environment_target

  if ! ${options["debug"]}; then
    set_environment_debug
  fi

  if ${options["concurrent-gc"]}; then
    set_environment_concurrent_gc
  fi

  if ${options["keep-failures"]}; then
    set_environment_keep_test_failures
  fi

  export SOONG_ALLOW_MISSING_DEPENDENCIES=true

  add_prebuilt_adb_dir_to_PATH

  # Set prebuilt_adb to be used in ART *.mk files, such as Android.mk
  set_global_ADB_to_prebuilt_adb
}

start_prebuilt_abd_server() {
  adb_kill_server
  safe "$(get_prebuilt_adb)" start-server
}

# Arguments:
#   ${1}: test type {run-test}
#   ${2}: a log file of 'run_test'
#   ${3}: bitness (32/64)
rerun_failed_tests() {
  local run_test_log_file="$2"
  if [[ -z "${run_test_log_file}" ]]; then
    log E "No log file of $1 was specified."
    return 1
  fi
  if [[ ! -f "${run_test_log_file}" ]]; then
    log E "File ${run_test_log_file} does not exist."
    return 1
  fi
  local failed_tests
  failed_tests=$(grep FAIL "${run_test_log_file}" | grep run-test | awk '{print $5}')
  # Note: $5 in the above line is not referencing an argument passed to this function.
  # It is referencing the 5th field in the log txt file, which is processed by awk.
  if [[ -z "${failed_tests}" ]]; then
    log E "Failed tests were not found in ${run_test_log_file}."
    return 1
  fi
  local return_code=0
  for failed_test in ${failed_tests}
  do
    log I "Rerunning test: ${failed_test}"
    run_test_unwrapped "single-test" "" "${failed_test}" "$3" \
      "target" "${job_count}" "$(declare -p options)"
    local new_return_code=$?
    if [[ ${return_code} -eq 0 ]]; then
      return_code=${new_return_code}
    fi
  done
  return ${return_code}
}

# Build (run) a test target using the Android build system.
# If a test failed, add it to the list of failed tests.
#
# If the option 'rerun-failed-tests' is set, failed tests will be rerun.
# Failed tests won't be rerun if there are any 'error: device' messages in logs.
# They mean a device has stability issues.
#
# Arguments:
#   ${1}: test type {run-test}
#   ${2}: test section
#   ${3}: bitness (32/64)
run_test() {
  local section_name=$2_$3
  if [[ -z "$3" ]]; then
    section_name+="32_64"
  fi

  # Check if we should skip section
  if ! ${options["$2"]}; then
    section_starter "${section_name}" "target"
    skip_section "${section_name}"
    return
  fi

  section_starter "${section_name}" "target"
  local run_test_log="$(get_workspace)/log_run_test_${section_name}.txt"
  run_test_unwrapped "$1" "$2" "" "$3" "target" "${job_count}" "$(declare -p options)"

  local return_code=$?
  if [[ ${return_code} -ne 0 ]] && ${options["rerun-failed-tests"]} ; then
    # Check the log file for errors related to the device, e.g.:
    #   error: device not found
    # They mean the device has stability issues caused a lot of tests to fail.
    # Rerunning such many tests is not feasible.
    if ! grep -q 'error: device' "${run_test_log}"; then
      # The device is stable enough to rerun tests.
      rerun_failed_tests "$1" "${run_test_log}" "$3"
      return_code=$?
    fi
  fi
  section_ender "${section_name}" "target" "${return_code}" "${options["keep-going"]}"
}

# Arguments:
#   ${1}: a path to gtest
run_gtest() {
  ${ADB} shell "chroot $ART_TEST_CHROOT \
                env ANDROID_ART_ROOT=$ART_TEST_APEX_ART_ROOT \
                    ANDROID_TZDATA_ROOT=$ART_TEST_APEX_TZDATA_ROOT \
                    ANDROID_I18N_ROOT=$ART_TEST_APEX_I18N_ROOT \
                    $1"
}

# Run gtests in parallel.
#
# Note: This is a temporary solution.
#
# Arguments:
#   ${1}: bitness (32/64)
test_gtest_parallel() {
  export -f run_gtest

  local joblog="$(get_workspace)/log_jobs_gtest_$1.txt"
  safe find_gtests "all" "target" | \
       parallel --jobs "${job_count}" --joblog "${joblog}" run_gtest {}
  local return_code=$?
  if [[ ${return_code} -ne 0 ]]; then
    # Format of log_jobs_gtest_$1.txt:
    #   Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
    failed_tests=$(awk '{if ($7 != 0) print $10}' "${joblog}"  | grep data)
    log E "Failed tests:\n${failed_tests}"
  fi
  return "${return_code}"
}

# Run gtests.
#
# Note: This is a temporary solution till testrunner.py supports to run
# gtest.
#
# Arguments:
#   ${1}: bitness (32/64)
# Disabling check for whether variable is declared before use.
# Check only disabled within the scope of test_gtest function.
# This is done as art_tools is defined in utils_run.sh, but used here.
# shellcheck disable=2154
test_gtest() {
  section_starter "gtest_$1" "target"

  if ! ${options["gtest"]}; then
    skip_section "gtest_$1"
    return
  fi

  if [[ -n "$(which parallel)" ]]; then
    test_gtest_parallel "$@"
  else
    "${art_tools}"/run-gtests.sh
  fi
  local -r return_code=$?

  section_ender "gtest_$1" "target" "${return_code}" "${options["keep-going"]}"
}

# Run ART tests with optimizing compiler.
# Arguments:
#   ${1}: bitness (32/64)
test_optimizing() {
  run_test "run-test" "optimizing" "$1"
}

# Run ART tests with interpreter.
# Arguments:
#   ${1}: bitness (32/64)
test_interpreter() {
  run_test "run-test" "interpreter" "$1"
}

# Run ART tests with JIT compiler.
# Arguments:
#   ${1}: bitness (32/64)
test_jit() {
  run_test "run-test" "jit" "$1"
}

# Run licore tests.
# Arguments:
#   ${1}: bitness (32/64)
test_libcore() {
  section_starter "libcore_$1" "target"

  if ! ${options["libcore"]}; then
    skip_section "libcore_$1"
    return
  fi

  local -a test_options=(--mode=device "--variant=X$1" --chroot "${ART_TEST_CHROOT}")
  if ${options["debug"]}; then
    test_options+=(--debug)
  fi

  "${art_tools}"/run-libcore-tests.sh "${test_options[@]}"
  local -r return_code=$?

  section_ender "libcore_$1" "target" "${return_code}" "${options[keep-going]}"
}
# Run jdwp tests.
# Arguments:
#   ${1}: bitness (32/64)
test_jdwp() {
  section_starter "jdwp_$1" "target"

  if ! ${options["jdwp"]}; then
    skip_section "jdwp_$1"
    return
  fi

  local -a test_options=(--mode=device "--variant=X$1" --chroot "${ART_TEST_CHROOT}")
  if ${options["debug"]}; then
    test_options+=(--debug)
  fi

  "${art_tools}"/run-jdwp-tests.sh "${test_options[@]}"
  local -r return_code=$?

  section_ender "jdwp_$1" "target" "${return_code}" "${options[keep-going]}"
}

set_job_count() {
  if [[ -n "${options["jobs"]}" ]]; then
    job_count="${options["jobs"]}"
  fi
}

main() {
  exit_on_failure arguments_parser options_format options -- "$@"

  if android_build_already_setup; then
    log E "This test does not support environment targets. Please re-run in a clean environment."
    exit 1
  fi

  if [[ ! -d "${PWD}/.repo" ]]; then
    log E "Script needs to be run at the root of the android tree."
    exit 1
  fi

  if [[ ! -f "$(get_prebuilt_adb)" ]]; then
    log E "$(get_prebuilt_adb) not found. Please check out prebuilt/runtime."
    exit 1
  fi

  # Set bitness options when they are not provided via the command line.
  # A single test name can contain the bitness at the end of its name.
  if ! ${options["32bit"]} && ! ${options["64bit"]}; then
    if [[ "${options["single-test"]}" = *"32" ]]; then
      options["32bit"]="true"
    elif [[ "${options["single-test"]}" = *"64" ]]; then
      options["64bit"]="true"
    else
      options["32bit"]="true"
      options["64bit"]="true"
    fi
  fi

  readonly options
  validate_options
  dump_options

  set_environment
  set_job_count

  start_test "${target_timer_name}"

  if ${options["restart-adb-server"]}; then
    start_prebuilt_abd_server
  fi
  select_android_target "arm" "$(retrieve_target_product_name)"

  for bits in 32 64; do
    if ! ${options["${bits}bit"]}; then
      log I "Skipping ${bits}bit tests."
      continue
    fi
    log I "Starting ${bits}bit tests."

    if ! ${options["skip-build"]}; then
      build_target "${bits}"
    else
      setup_android_target_from_bits "${bits}"
    fi

    add_prebuilt_adb_dir_to_PATH
    buildbot_device_prepare "${bits}"

    if [[ -n ${options["single-test"]} ]]; then
      test_single "${bits}" "target" "${job_count}" "$(declare -p options)"
      continue
    fi

    test_gtest "${bits}"
    test_optimizing "${bits}"
    test_interpreter "${bits}"
    test_jit "${bits}"
    test_libcore "${bits}"
    test_jdwp "${bits}"
  done

  end_test "${target_timer_name}"
}

main "$@"