binary search tool: Add object bisecting to bisect.py

Add object file bisecting to bisect.py. This is the frontend to the
sysroot_wrapper bisection scripts. This bisection mode takes the
following arguments: board, remote IP, package to bisect, and optionally
the directory for the good/bad build trees.

Adds default install/test scripts to sysroot_wrapper (similar to
default scripts in cros_pkg) to support this bisection mode.

TEST=Run unit tests and system test with cryptohome package
CQ-DEPEND=CL:*267995

Change-Id: I1e9166b753f78ca7bdcf6ddbd056af62bc03923b
Reviewed-on: https://chrome-internal-review.googlesource.com/268035
Commit-Ready: Cassidy Burden <cburden@google.com>
Tested-by: Cassidy Burden <cburden@google.com>
Reviewed-by: Caroline Tice <cmtice@google.com>

5 files changed, 319 insertions, 5 deletions

diff --git a/binary_search_tool/README.bisect b/binary_search_tool/README.bisect
index 904fd7f2..ab83118c 100644
--- a/binary_search_tool/README.bisect
+++ b/binary_search_tool/README.bisect
@@ -40,7 +40,7 @@ Bisection Methods:
     --switch_to_bad='cros_pkg/switch_to_bad.sh'
     --install_script='cros_pkg/install.sh'
     --test_script='cros_pkg/interactive_test.sh'
-    --noincremental
+    --incremental
     --prune
     --file_args
 
@@ -56,7 +56,64 @@ Bisection Methods:
         ./bisect.py package daisy 172.17.211.184 -t cros_pkg/boot_test.sh
 
 2) ChromeOS Object:
-  WIP
+  This method will bisect across all objects in a ChromeOS package and find
+  the offending objects (according to your test script). This method takes the
+  following arguments:
+
+    board: The board to bisect on. For example: daisy, falco, etc.
+    remote: The IP address of the physical machine you're using to test with.
+    package: The package to bisect with. For example: chromeos-chrome
+    dir: (Optional) the directory for your good/bad build trees. Defaults to
+         $BISECT_DIR or /tmp/sysroot_bisect. This value will set $BISECT_DIR
+         for all bisecting scripts.
+
+  By default the ChromeOS object method will do a simple interactive test that
+  pings the machine and prompts the user if the machine is good.
+
+  a) Setup:
+    The ChromeOS package method requires that you populate your good and bad set
+    of objects. sysroot_wrapper will automatically detect the BISECT_STAGE
+    variable and use this to populate emerged objects. Here is an example:
+
+      # Defaults to /tmp/sysroot_bisect
+      BISECT_DIR="/path/to/where/you/want/to/store/builds/"
+
+      BISECT_STAGE="POPULATE_GOOD"
+      ./switch_to_good_compiler.sh
+      emerge-${board} -C ${package_to_bisect}
+      emerge-${board} ${package_to_bisect}
+
+      BISECT_STAGE="POPULATE_BAD"
+      ./switch_to_bad_compiler.sh
+      emerge-${board} -C {package_to_bisect}
+      emerge-${board} ${package_to_bisect}
+
+  b) Cleanup:
+    The user must clean up all built images and the populated object files.
+
+  c) Default Arguments:
+    --get_initial_items='sysroot_wrapper/get_initial_items.sh'
+    --switch_to_good='sysroot_wrapper/switch_to_good.sh'
+    --switch_to_bad='sysroot_wrapper/switch_to_bad.sh'
+    --install_script='sysroot_wrapper/install.sh'
+    --test_script='sysroot_wrapper/interactive_test.sh'
+    --noincremental
+    --prune
+    --file_args
+
+  d) Additional Documentation:
+    See ./sysroot_wrapper/README for full documentation of ChromeOS object file
+    bisecting.
+
+  e) Examples:
+    i)  Basic interactive test object bisection, on daisy board for
+        cryptohome package:
+        ./bisect.py object daisy 172.17.211.184 cryptohome
+
+    ii) Basic boot test package bisection, on daisy board for cryptohome
+        package:
+        ./bisect.py object daisy 172.17.211.184 cryptohome \
+        --test_script=sysroot_wrapper/boot_test.sh
 
 Resuming:
   bisect.py and binary_search_state.py offer the ability to resume a bisection
diff --git a/binary_search_tool/bisect.py b/binary_search_tool/bisect.py
index 027956e5..8e454f62 100755
--- a/binary_search_tool/bisect.py
+++ b/binary_search_tool/bisect.py
@@ -168,17 +168,46 @@ class BisectPackage(Bisector):
 class BisectObject(Bisector):
   """The class for object bisection steps."""
 
+  sysroot_wrapper_files = ['bad/_LIST', 'good/_LIST']
+
   def __init__(self, options, overrides):
     super(BisectObject, self).__init__(options, overrides)
+    self.method_name = 'ChromeOS Object'
+    self.default_kwargs = {
+        'get_initial_items': 'sysroot_wrapper/get_initial_items.sh',
+        'switch_to_good': 'sysroot_wrapper/switch_to_good.sh',
+        'switch_to_bad': 'sysroot_wrapper/switch_to_bad.sh',
+        'install_script': 'sysroot_wrapper/install.sh',
+        'test_script': 'sysroot_wrapper/interactive_test.sh',
+        'noincremental': True,
+        'prune': True,
+        'file_args': True
+    }
+    self.options = options
+    if options.dir:
+      os.environ['BISECT_DIR'] = options.dir
+    self.options.dir = os.environ.get('BISECT_DIR', '/tmp/sysroot_bisect')
+
+    self.ArgOverride(self.default_kwargs, overrides)
 
   def PreRun(self):
-    raise NotImplementedError('Object bisecting still WIP')
+    for f in self.sysroot_wrapper_files:
+      full_path = os.path.join(self.options.dir, f)
+      if not os.path.exists(full_path):
+        self.logger.LogError('Object bisector setup failed, %s does not exist' %
+                             full_path)
+        return 1
+
+    os.environ['BISECT_BOARD'] = self.options.board
+    os.environ['BISECT_REMOTE'] = self.options.remote
+    os.environ['BISECT_PACKAGE'] = self.options.package
+    return 0
 
   def Run(self):
-    return 1
+    return binary_search_state.Run(**self.default_kwargs)
 
   def PostRun(self):
-    return 1
+    return 0
 
 
 def Run(bisector):
@@ -236,6 +265,15 @@ def Main(argv):
   parser_package.set_defaults(handler=BisectPackage)
 
   parser_object = subparsers.add_parser('object')
+  parser_object.add_argument('board', help='Board to target')
+  parser_object.add_argument('remote', help='Remote machine to test on')
+  parser_object.add_argument('package', help='Package to emerge and test')
+  parser_object.add_argument('--dir',
+                             help=('Bisection directory to use, sets '
+                                   '$BISECT_DIR if provided. Defaults to '
+                                   'current value of $BISECT_DIR (or '
+                                   '/tmp/sysroot_bisect if $BISECT_DIR is '
+                                   'empty).'))
   parser_object.set_defaults(handler=BisectObject)
 
   options, remaining = parser.parse_known_args(argv)
diff --git a/binary_search_tool/sysroot_wrapper/boot_test.sh b/binary_search_tool/sysroot_wrapper/boot_test.sh
new file mode 100755
index 00000000..a3de2dec
--- /dev/null
+++ b/binary_search_tool/sysroot_wrapper/boot_test.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -u
+#
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# This script pings the chromebook to determine if it has successfully booted.
+#
+# This script is intended to be used by binary_search_state.py, as
+# part of the binary search triage on ChromeOS object files. It waits for the
+# install script to build and install the image, then pings the machine.
+# It should return '0' if the test succeeds (the image booted); '1' if the test
+# fails (the image did not boot); and '2' if it could not determine (does not
+# apply in this case).
+#
+
+source sysroot_wrapper/common.sh
+
+# Send 3 pings and wait 3 seconds for any responsed (then timeout).
+ping -c 3 -W 3 ${BISECT_REMOTE}
+retval=$?
+
+
+exit $retval
diff --git a/binary_search_tool/sysroot_wrapper/install.sh b/binary_search_tool/sysroot_wrapper/install.sh
new file mode 100755
index 00000000..7cf62168
--- /dev/null
+++ b/binary_search_tool/sysroot_wrapper/install.sh
@@ -0,0 +1,160 @@
+#!/bin/bash
+#
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# This script calls build_image to generate a new ChromeOS image,
+# using whatever packages are currently in the build tree.  If
+# build_images succeeeds, then it pushes the new ChromeOS image onto a
+# chromebook. If pushing the ChromeOS image onto the chromebook fails it
+# will walk the user through troubleshooting the problem.
+#
+# This script is intended to be used by binary_search_state.py, as
+# part of the binary search triage on ChromeOS objects. It should return '0'
+# if the install succeeds; and '1' if the install fails (the image could not
+# build or be flashed).
+#
+# This script requires $BISECT_BOARD, $BISECT_REMOTE, and $BISECT_PACKAGE to be
+# set. If they are not set the script will exit.
+#
+
+export PYTHONUNBUFFERED=1
+
+source sysroot_wrapper/common.sh
+
+usb_flash()
+{
+  echo
+  echo "Insert a usb stick into the current machine"
+  echo "Note: The cros flash will take time and doesn't give much output."
+  echo "      Be patient. If your usb access light is flashing it's working."
+  sleep 1
+  read -p "Press enter to continue" notused
+
+  cros flash --board=${BISECT_BOARD} --clobber-stateful usb:// ~/trunk/src/build/images/${BISECT_BOARD}/latest/chromiumos_test_image.bin
+
+  echo
+  echo "Flash to usb complete!"
+  echo "Plug the usb into your chromebook and install the image."
+  echo "Refer to the ChromiumOS Developer's Handbook for more details."
+  echo "http://www.chromium.org/chromium-os/developer-guide#TOC-Boot-from-your-USB-disk"
+  while true; do
+    sleep 1
+    read -p "Was the installation of the image successful? " choice
+    case $choice in
+        [Yy]*) return 0;;
+        [Nn]*) return 1;;
+        *) echo "Please answer y or n.";;
+    esac
+  done
+}
+
+ethernet_flash()
+{
+  echo
+  echo "Please ensure your Chromebook is up and running Chrome so"
+  echo "cros flash may run."
+  echo "If your Chromebook has a broken image you can try:"
+  echo "1. Rebooting your Chromebook 6 times to install the last working image"
+  echo "2. Alternatively, running the following command on the Chromebook"
+  echo "   will also rollback to the last working image:"
+  echo "   'update_engine_client --rollback --nopowerwash --reboot'"
+  echo "3. Flashing a new image through USB"
+  echo
+  sleep 1
+  read -p $'Press enter to continue and retry the ethernet flash' notused
+  cros flash --board=${BISECT_BOARD} --clobber-stateful ${BISECT_REMOTE} ~/trunk/src/build/images/${BISECT_BOARD}/latest/chromiumos_test_image.bin
+}
+
+reboot()
+{
+  ret_val=0
+  pushd ~/trunk/src/scripts > /dev/null
+  set -- --remote=${BISECT_REMOTE}
+  . ./common.sh || ret_val=1
+  . ./remote_access.sh || ret_val=1
+  TMP=$(mktemp -d)
+  FLAGS "$@" || ret_val=1
+  remote_access_init || ret_val=1
+  remote_reboot || ret_val=1
+  popd > /dev/null
+
+  return $ret_val
+}
+
+if [[ -z "${BISECT_BOARD}" || -z "${BISECT_REMOTE}" || -z "${BISECT_PACKAGE}" ]]
+then
+  echo "ERROR: Environment variables not properly setup. Please ensure that "
+  echo "\$BISECT_BOARD, \$BISECT_REMOTE, and \$BISECT_PACKAGE are set"
+  exit 3
+fi
+
+echo
+echo "INSTALLATION BEGIN"
+echo
+echo "EMERGING ${BISECT_PACKAGE}"
+CLEAN_DELAY=0 emerge-${BISECT_BOARD} -C ${BISECT_PACKAGE}
+emerge-${BISECT_BOARD} ${BISECT_PACKAGE}
+emerge_status=$?
+
+if [[ ${emerge_status} -ne 0 ]] ; then
+  echo "emerging ${BISECT_PACKAGE} returned a non-zero status: $emerge_status"
+  exit 1
+fi
+
+echo
+echo "DEPLOYING"
+echo "cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE}"
+cros deploy ${BISECT_REMOTE} ${BISECT_PACKAGE} --log-level=info
+deploy_status=$?
+
+if [[ ${deploy_status} -eq 0 ]] ; then
+  echo "Deploy successful. Rebooting device..."
+  reboot
+  if [[ $? -ne 0 ]] ; then
+    echo
+    echo "Could not automatically reboot device!"
+    read -p "Please manually reboot device and press enter to continue" notused
+  fi
+  exit 0
+fi
+
+echo "Deploy failed! Trying build_image/cros flash instead..."
+echo
+echo "BUILDING IMAGE"
+pushd ~/trunk/src/scripts
+./build_image test --board=${BISECT_BOARD} --noenable_rootfs_verification
+build_status=$?
+popd
+
+if [[ ${build_status} -eq 0 ]] ; then
+    echo
+    echo "FLASHING"
+    echo "Pushing built image onto device."
+    echo "cros flash --board=${BISECT_BOARD} --clobber-stateful ${BISECT_REMOTE} ~/trunk/src/build/images/${BISECT_BOARD}/latest/chromiumos_test_image.bin"
+    cros flash --board=${BISECT_BOARD} --clobber-stateful ${BISECT_REMOTE} ~/trunk/src/build/images/${BISECT_BOARD}/latest/chromiumos_test_image.bin
+    cros_flash_status=$?
+    while [[ ${cros_flash_status} -ne 0 ]] ; do
+        while true; do
+          echo
+          echo "cros flash has failed! From here you can:"
+          echo "1. Flash through USB"
+          echo "2. Retry flashing over ethernet"
+          echo "3. Abort this installation and skip this image"
+          sleep 1
+          read -p "Which method would you like to do? " choice
+          case $choice in
+              1) usb_flash && break;;
+              2) ethernet_flash && break;;
+              3) exit 1;;
+              *) echo "Please answer 1, 2, or 3.";;
+          esac
+        done
+
+        cros_flash_status=$?
+    done
+else
+    echo "build_image returned a non-zero status: ${build_status}"
+    exit 1
+fi
+
+exit 0
diff --git a/binary_search_tool/sysroot_wrapper/interactive_test.sh b/binary_search_tool/sysroot_wrapper/interactive_test.sh
new file mode 100755
index 00000000..1b5b7e19
--- /dev/null
+++ b/binary_search_tool/sysroot_wrapper/interactive_test.sh
@@ -0,0 +1,37 @@
+#!/bin/bash -u
+#
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# This script pings the chromebook to determine if it successfully booted.
+# It then asks the user if the image is good or not, allowing the user to
+# conduct whatever tests the user wishes, and waiting for a response.
+#
+# This script is intended to be used by binary_search_state.py, as
+# part of the binary search triage on ChromeOS object files. It waits for the
+# install script to build and install the image, then asks the user if the
+# image is good or not. It should return '0' if the test succeeds (the image
+# is 'good'); '1' if the test fails (the image is 'bad'); and '2' if it could
+# not determine (does not apply in this case).
+#
+
+source cros_pkg/common.sh
+
+ping -c 3 -W 3 ${BISECT_REMOTE}
+retval=$?
+
+if [[ ${retval} -eq 0 ]]; then
+    echo "ChromeOS image has been built and installed on ${BISECT_REMOTE}."
+else
+    exit 1
+fi
+
+while true; do
+    read -p "Is this a good ChromeOS image?" yn
+    case $yn in
+        [Yy]* ) exit 0;;
+        [Nn]* ) exit 1;;
+        * ) echo "Please answer yes or no.";;
+    esac
+done
+
+exit 2