CameraITS: Updated the automated running script

Change-Id: Id173a3408c13b60da55b6738fb22023549e05ee6

4 files changed, 76 insertions, 108 deletions

diff --git a/apps/CameraITS/README b/apps/CameraITS/README
index 191365b..c7b786c 100644
--- a/apps/CameraITS/README
+++ b/apps/CameraITS/README
@@ -200,16 +200,15 @@ camera, and camera=1 is the front-facing camera.
 
     python tests/scene1/test_linearity.py camera=1
 
-The tools/run_scene_tests.sh script can be executed from inside one of the
-tests/scene<N> folders, and it will simply run each test script in that folder
-in turn, reporting whether it passed or failed. The scene should be setup as
-described in the README before running this script. It will create a sub-dir
-named "out" where the files created by the various tests will end up. This
-script can take the same "reboot" and "camera" args as the test scripts.
-
-    cd tests/scene1/
-    ../../tools/run_scene_tests.sh
-    ../../tools/run_scene_tests.sh reboot=30 camera=1
+The tools/run_all_tests.py script should be executed from the top-level
+CameraITS directory, and it will run all of the tests in an automated fashion,
+saving the generated output files along with the stdout and stderr dumps to
+a temporary directory.
+
+    python tools/run_all_tests.py
+
+This can be run with the "noinit" argument, and in general any args provided
+to this command line will be passed to each script as it is executed.
 
 The tests/inprog directory contains a mix of unfinished, in-progress, and
 incomplete tests. These may or may not be useful in testing a HAL impl.,
diff --git a/apps/CameraITS/pymodules/its/device.py b/apps/CameraITS/pymodules/its/device.py
index 2cefad1..271e899 100644
--- a/apps/CameraITS/pymodules/its/device.py
+++ b/apps/CameraITS/pymodules/its/device.py
@@ -90,6 +90,7 @@ class ItsSession(object):
                 print "Reboot complete"
 
         # TODO: Figure out why "--user 0" is needed, and fix the problem.
+        _run('%s shell am force-stop --user 0 %s' % (adb, self.PACKAGE))
         _run(('%s shell am startservice --user 0 -t text/plain '
               '-a %s') % (adb, self.INTENT_START))
 
diff --git a/apps/CameraITS/tools/run_all_tests.py b/apps/CameraITS/tools/run_all_tests.py
new file mode 100644
index 0000000..4677331
--- /dev/null
+++ b/apps/CameraITS/tools/run_all_tests.py
@@ -0,0 +1,66 @@
+# Copyright 2014 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path
+import tempfile
+import subprocess
+import time
+import sys
+
+def main():
+    """Run all the automated tests, saving intermediate files, and producing
+    a summary/report of the results.
+
+    Script should be run from the top-level CameraITS directory.
+    """
+
+    # Get all the scene0 and scene1 tests, which can be run using the same
+    # physical setup.
+    scenes = ["scene0", "scene1"]
+    tests = []
+    for d in scenes:
+        tests += [(d,s[:-3],os.path.join("tests", d, s))
+                  for s in os.listdir(os.path.join("tests",d))
+                  if s[-3:] == ".py"]
+    tests.sort()
+
+    # Make output directories to hold the generated files.
+    topdir = tempfile.mkdtemp()
+    for d in scenes:
+        os.mkdir(os.path.join(topdir, d))
+    print "Saving output files to:", topdir, "\n"
+
+    # Run each test, capturing stdout and stderr.
+    numpass = 0
+    for (scene,testname,testpath) in tests:
+        cmd = ['python', os.path.join(os.getcwd(),testpath)] + sys.argv[1:]
+        outdir = os.path.join(topdir,scene)
+        outpath = os.path.join(outdir,testname+"_stdout.txt")
+        errpath = os.path.join(outdir,testname+"_stderr.txt")
+        t0 = time.time()
+        with open(outpath,"w") as fout, open(errpath,"w") as ferr:
+            retcode = subprocess.call(cmd,stderr=ferr,stdout=fout,cwd=outdir)
+        t1 = time.time()
+        print "%s %s/%s [%.1fs]" % (
+                "PASS" if retcode==0 else "FAIL", scene, testname, t1-t0)
+        if retcode == 0:
+            numpass += 1
+
+    print "\n%d / %d tests passed (%.1f%%)" % (
+            numpass, len(tests), 100.0*float(numpass)/len(tests))
+
+if __name__ == '__main__':
+    main()
+
diff --git a/apps/CameraITS/tools/run_scene_tests.sh b/apps/CameraITS/tools/run_scene_tests.sh
deleted file mode 100755
index c31f36e..0000000
--- a/apps/CameraITS/tools/run_scene_tests.sh
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# -----------------------------------------------------------------------------
-# The tests exercised in this file all assert/exit on failure, and terminate
-# cleanly on success.
-
-# -----------------------------------------------------------------------------
-# This script should be run from inside a tests/scene<N> directory.
-
-if [ ! -f ../../tools/config.py ]
-then
-    echo "This script must be run from inside a tests/scene<N> directory."
-    exit
-fi
-
-# By default, run the tests on the main/rear camera.
-CAMERA=0
-
-for i in "$@"
-do
-    case $i in
-        camera=*)
-        # Run the tests on the specified camera ID.
-        CAMERA="${i#*=}"
-        shift
-        ;;
-
-        reboot)
-        # Reboot the device prior to running each test, to ensure that a
-        # problem in one test doesn't propagate into subsequent tests; use this
-        # when debugging test failures, but not when running the test suite
-        # (since all tests should pass even when run back-to-back).
-        REBOOT=reboot
-        ;;
-
-        reboot=*)
-        # Reboot and wait the given duration.
-        REBOOT="reboot=${i#*=}"
-        shift
-        ;;
-
-        *)
-        # Unknown option.
-        ;;
-    esac
-done
-
-rm -rf out
-mkdir -p out
-
-echo Running tests with args: $REBOOT camera=$CAMERA target noinit
-
-cd out
-python ../../../tools/config.py $REBOOT camera=$CAMERA
-cd ..
-
-testcount=0
-failcount=0
-
-for T in *.py
-do
-    cd out
-    let testcount=testcount+1
-    echo ""
-    echo "--------------------------------------------------------------------"
-    echo "Running test: $T"
-    echo "--------------------------------------------------------------------"
-    python ../"$T" $REBOOT camera=$CAMERA noinit target
-    code=$?
-    if [ $code -ne 0 ]; then
-        let failcount=failcount+1
-        echo ""
-        echo "###############"
-        echo "# Test failed #"
-        echo "###############"
-    fi
-    echo ""
-    cd ..
-done
-
-echo ""
-echo "$failcount out of $testcount tests failed"
-echo ""
-