CameraITS: Restructured test folders to group by scenes.

Change-Id: I3dc66bd44569f4416826d7f23b4e63533058bd3f

31 files changed, 101 insertions, 212 deletions

diff --git a/apps/CameraITS/README b/apps/CameraITS/README
index a2aa9f9..250234d 100644
--- a/apps/CameraITS/README
+++ b/apps/CameraITS/README
@@ -174,24 +174,35 @@ for example:
 
     mkdir out
     cd out
-    python ../tests/test_linearity.py
+    python ../tests/scene1/test_linearity.py
 
 Any test can be specified to reboot the camera prior to capturing any shots, by
 adding a "reboot" or "reboot=N" command line argument, where N is the number of
 seconds to wait after rebooting the device before sending any commands; the
 default is 30 seconds.
 
-    python tests/test_linearity.py reboot
-    python tests/test_linearity.py reboot=20
+    python tests/scene1/test_linearity.py reboot
+    python tests/scene1/test_linearity.py reboot=20
 
 It's possible that a test could leave the camera in a bad state, in particular
 if there are any bugs in the HAL or the camera framework. Rebooting the device
 can be used to get it into a known clean state again.
 
+Each test assumes some sort of target or scene. There are multiple scene<N>
+folders under the tests directory, and each contains a README file which
+describes the scene for the scripts in that folder.
+
+The tools/run_scene_tests.sh script can be executed from inside one of the
+tests/scene<N> folders, and it will simply run each test script in that folder
+in turn, reporting whether it passed or failed. The scene should be setup as
+described in the README before running this script.
+
+    ../../tools/run_scene_tests.sh
+
 3.4. Target exposure
 --------------------
 
-The tests/config.py script is a wrapper for the its.target module, which is
+The tools/config.py script is a wrapper for the its.target module, which is
 used to set an exposure level based on the scene that the camera is imaging.
 The purpose of this is to be able to have tests which use hard-coded manual
 exposure controls, while at the same time ensuring that the captured images
@@ -201,8 +212,8 @@ If no argument is provided, the script will use the camera to measure the
 scene to determine the exposure level. An argument can be provided to hard-
 code the exposure level.
 
-    python tests/config.py
-    python tests/config.py 16531519962
+    python tools/config.py
+    python tools/config.py 16531519962
 
 This creates a file named its.target.cfg in the current directory, storing the
 target exposure level. Tests that use the its.target module will be reusing
diff --git a/apps/CameraITS/tests/regress.sh b/apps/CameraITS/tests/regress.sh
deleted file mode 100755
index b88e0af..0000000
--- a/apps/CameraITS/tests/regress.sh
+++ /dev/null
@@ -1,106 +0,0 @@
-#!/bin/bash
-
-# Copyright 2013 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The tests exercised in this file all assert/exit on failure, and terminate
-# cleanly on success.
-
-# Optionally, the device can be rebooted for each test, to ensure that
-# a problem in one test doesn't propagate into subsequent tests; use this
-# when debugging test failures, but not when running the test suite (since
-# all tests should pass even when run back-to-back).
-#REBOOT=reboot
-
-echo ""
-echo "--------------------------------------------------------------------"
-echo "Set up camera for scene 1:"
-echo "* Camera on tripod, in either portrait or landscape orientation"
-echo "* Scene filled (or mostly filled) by grey card"
-echo "* Illuminated by simple light source, for example a desk lamp"
-echo "* Uniformity of lighting and target positioning need not be precise"
-echo ""
-echo "Use any camera app to be able to see preview while setting up the"
-echo "camera, grey card, and light source; make sure that the camera app"
-echo "is killed before proceeding."
-echo ""
-echo "Press ENTER when the camera set up is complete ..."
-echo "--------------------------------------------------------------------"
-read DUMMY
-
-rm -rf out1
-mkdir -p out1
-cd out1
-
-python ../config.py $REBOOT
-
-testcount=0
-failcount=0
-
-for T in \
-         test_3a.py \
-         test_black_white.py \
-         test_camera_properties.py \
-         test_capture_result.py \
-         test_exposure.py \
-         test_formats.py \
-         test_jitter.py \
-         test_jpeg.py \
-         test_param_color_correction.py \
-         test_param_exposure_time.py \
-         test_param_noise_reduction.py \
-         test_param_sensitivity.py \
-         test_param_sensitivity_burst.py \
-         test_param_tonemap_mode.py \
-         test_yuv_plus_dng.py \
-         test_yuv_plus_jpeg.py \
-         test_yuv_plus_raw.py \
-         \
-         test_auto.py \
-         test_crop_region.py \
-         test_format_combos.py \
-         test_latching.py \
-         test_linearity.py \
-         test_long_burst.py \
-         test_param_edge_mode.py \
-         test_param_flash_mode.py \
-         test_test_patterns.py \
-
-         #test_blc_lsc.py \
-         #test_dng_tags.py \
-
-do
-    let testcount=testcount+1
-    echo ""
-    echo "--------------------------------------------------------------------"
-    echo "Running test: $T"
-    echo "--------------------------------------------------------------------"
-    python ../"$T" $REBOOT
-    code=$?
-    if [ $code -ne 0 ]; then
-        let failcount=failcount+1
-        echo ""
-        echo "###############"
-        echo "# Test failed #"
-        echo "###############"
-    fi
-    echo ""
-done
-
-echo ""
-echo "$failcount out of $testcount tests failed"
-echo ""
-
-cd ..
-
diff --git a/apps/CameraITS/tests/run1.sh b/apps/CameraITS/tests/run1.sh
deleted file mode 100755
index 3e35865..0000000
--- a/apps/CameraITS/tests/run1.sh
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/bin/bash
-
-# Copyright 2013 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The tests exercised in this file all assert/exit on failure, and terminate
-# cleanly on success.
-
-# Optionally, the device can be rebooted for each test, to ensure that
-# a problem in one test doesn't propagate into subsequent tests; use this
-# when debugging test failures, but not when running the test suite (since
-# all tests should pass even when run back-to-back).
-#REBOOT=reboot
-
-echo ""
-echo "--------------------------------------------------------------------"
-echo "Set up camera for scene 1:"
-echo "* Camera on tripod in portrait orientation"
-echo "* Scene filled (or mostly filled) by grey card"
-echo "* Illuminated by simple light source, for example a desk lamp"
-echo "* Uniformity of lighting and target positioning need not be precise"
-echo ""
-echo "Use any camera app to be able to see preview while setting up the"
-echo "camera, grey card, and light source; make sure that the camera app"
-echo "is killed before proceeding."
-echo ""
-echo "Press ENTER when the camera set up is complete ..."
-echo "--------------------------------------------------------------------"
-read DUMMY
-
-rm -rf out1
-mkdir -p out1
-cd out1
-
-python ../config.py $REBOOT
-
-testcount=0
-failcount=0
-
-for T in \
-         test_3a.py \
-         test_auto.py \
-         test_black_white.py \
-         test_camera_properties.py \
-         test_capture_result.py \
-         test_exposure.py \
-         test_formats.py \
-         test_format_combos.py \
-         test_jitter.py \
-         test_jpeg.py \
-         test_latching.py \
-         test_linearity.py \
-         test_param_color_correction.py \
-         test_param_exposure_time.py \
-         test_param_flash_mode.py \
-         test_param_noise_reduction.py \
-         test_param_sensitivity.py \
-         test_param_sensitivity_burst.py \
-         test_param_tonemap_mode.py \
-         test_sensor_events.py \
-         test_unified_timestamps.py \
-         test_yuv_plus_dng.py \
-         test_yuv_plus_jpeg.py \
-         test_yuv_plus_raw.py \
-
-do
-    let testcount=testcount+1
-    echo ""
-    echo "--------------------------------------------------------------------"
-    echo "Running test: $T"
-    echo "--------------------------------------------------------------------"
-    python ../"$T" $REBOOT
-    code=$?
-    if [ $code -ne 0 ]; then
-        let failcount=failcount+1
-        echo ""
-        echo "###############"
-        echo "# Test failed #"
-        echo "###############"
-    fi
-    echo ""
-done
-
-echo ""
-echo "$failcount out of $testcount tests failed"
-echo ""
-
-cd ..
-
diff --git a/apps/CameraITS/tests/scene1/README b/apps/CameraITS/tests/scene1/README
new file mode 100755
index 0000000..154c2af
--- /dev/null
+++ b/apps/CameraITS/tests/scene1/README
@@ -0,0 +1,9 @@
+Scene 1 description:
+* Camera on tripod in portrait orientation
+* Scene filled (or mostly filled) by grey card
+* Illuminated by simple light source, for example a desk lamp
+* Uniformity of lighting and target positioning need not be precise
+
+This is intended to be a very simple setup that can be recreated on
+an engineer's desk without any large or expensive equipment.
+
diff --git a/apps/CameraITS/tests/test_3a.py b/apps/CameraITS/tests/scene1/test_3a.py
index eaeab0b..eaeab0b 100644
--- a/apps/CameraITS/tests/test_3a.py
+++ b/apps/CameraITS/tests/scene1/test_3a.py
diff --git a/apps/CameraITS/tests/test_auto.py b/apps/CameraITS/tests/scene1/test_auto.py
index 14b7f89..14b7f89 100644
--- a/apps/CameraITS/tests/test_auto.py
+++ b/apps/CameraITS/tests/scene1/test_auto.py
diff --git a/apps/CameraITS/tests/test_black_white.py b/apps/CameraITS/tests/scene1/test_black_white.py
index 64d6045..64d6045 100644
--- a/apps/CameraITS/tests/test_black_white.py
+++ b/apps/CameraITS/tests/scene1/test_black_white.py
diff --git a/apps/CameraITS/tests/test_camera_properties.py b/apps/CameraITS/tests/scene1/test_camera_properties.py
index 5f3b165..5f3b165 100644
--- a/apps/CameraITS/tests/test_camera_properties.py
+++ b/apps/CameraITS/tests/scene1/test_camera_properties.py
diff --git a/apps/CameraITS/tests/test_capture_result.py b/apps/CameraITS/tests/scene1/test_capture_result.py
index 2e4d374..2e4d374 100644
--- a/apps/CameraITS/tests/test_capture_result.py
+++ b/apps/CameraITS/tests/scene1/test_capture_result.py
diff --git a/apps/CameraITS/tests/test_exposure.py b/apps/CameraITS/tests/scene1/test_exposure.py
index 82919e9..82919e9 100644
--- a/apps/CameraITS/tests/test_exposure.py
+++ b/apps/CameraITS/tests/scene1/test_exposure.py
diff --git a/apps/CameraITS/tests/test_format_combos.py b/apps/CameraITS/tests/scene1/test_format_combos.py
index 7e45197..7e45197 100644
--- a/apps/CameraITS/tests/test_format_combos.py
+++ b/apps/CameraITS/tests/scene1/test_format_combos.py
diff --git a/apps/CameraITS/tests/test_formats.py b/apps/CameraITS/tests/scene1/test_formats.py
index 16fc3fd..16fc3fd 100644
--- a/apps/CameraITS/tests/test_formats.py
+++ b/apps/CameraITS/tests/scene1/test_formats.py
diff --git a/apps/CameraITS/tests/test_jitter.py b/apps/CameraITS/tests/scene1/test_jitter.py
index 9fb20d1..9fb20d1 100644
--- a/apps/CameraITS/tests/test_jitter.py
+++ b/apps/CameraITS/tests/scene1/test_jitter.py
diff --git a/apps/CameraITS/tests/test_jpeg.py b/apps/CameraITS/tests/scene1/test_jpeg.py
index 54a6189..54a6189 100644
--- a/apps/CameraITS/tests/test_jpeg.py
+++ b/apps/CameraITS/tests/scene1/test_jpeg.py
diff --git a/apps/CameraITS/tests/test_latching.py b/apps/CameraITS/tests/scene1/test_latching.py
index 4b11aef..4b11aef 100644
--- a/apps/CameraITS/tests/test_latching.py
+++ b/apps/CameraITS/tests/scene1/test_latching.py
diff --git a/apps/CameraITS/tests/test_linearity.py b/apps/CameraITS/tests/scene1/test_linearity.py
index 9e02e62..9e02e62 100644
--- a/apps/CameraITS/tests/test_linearity.py
+++ b/apps/CameraITS/tests/scene1/test_linearity.py
diff --git a/apps/CameraITS/tests/test_param_color_correction.py b/apps/CameraITS/tests/scene1/test_param_color_correction.py
index 7764026..7764026 100644
--- a/apps/CameraITS/tests/test_param_color_correction.py
+++ b/apps/CameraITS/tests/scene1/test_param_color_correction.py
diff --git a/apps/CameraITS/tests/test_param_exposure_time.py b/apps/CameraITS/tests/scene1/test_param_exposure_time.py
index b982bdc..b982bdc 100644
--- a/apps/CameraITS/tests/test_param_exposure_time.py
+++ b/apps/CameraITS/tests/scene1/test_param_exposure_time.py
diff --git a/apps/CameraITS/tests/test_param_flash_mode.py b/apps/CameraITS/tests/scene1/test_param_flash_mode.py
index a1aaf92..a1aaf92 100644
--- a/apps/CameraITS/tests/test_param_flash_mode.py
+++ b/apps/CameraITS/tests/scene1/test_param_flash_mode.py
diff --git a/apps/CameraITS/tests/test_param_noise_reduction.py b/apps/CameraITS/tests/scene1/test_param_noise_reduction.py
index d183630..d183630 100644
--- a/apps/CameraITS/tests/test_param_noise_reduction.py
+++ b/apps/CameraITS/tests/scene1/test_param_noise_reduction.py
diff --git a/apps/CameraITS/tests/test_param_sensitivity.py b/apps/CameraITS/tests/scene1/test_param_sensitivity.py
index 43aec76..43aec76 100644
--- a/apps/CameraITS/tests/test_param_sensitivity.py
+++ b/apps/CameraITS/tests/scene1/test_param_sensitivity.py
diff --git a/apps/CameraITS/tests/test_param_sensitivity_burst.py b/apps/CameraITS/tests/scene1/test_param_sensitivity_burst.py
index 2df1d08..2df1d08 100644
--- a/apps/CameraITS/tests/test_param_sensitivity_burst.py
+++ b/apps/CameraITS/tests/scene1/test_param_sensitivity_burst.py
diff --git a/apps/CameraITS/tests/test_param_tonemap_mode.py b/apps/CameraITS/tests/scene1/test_param_tonemap_mode.py
index 919d279..919d279 100644
--- a/apps/CameraITS/tests/test_param_tonemap_mode.py
+++ b/apps/CameraITS/tests/scene1/test_param_tonemap_mode.py
diff --git a/apps/CameraITS/tests/test_raw_sensitivity.py b/apps/CameraITS/tests/scene1/test_raw_sensitivity.py
index d01ab7b..d34edf7 100644
--- a/apps/CameraITS/tests/test_raw_sensitivity.py
+++ b/apps/CameraITS/tests/scene1/test_raw_sensitivity.py
@@ -40,6 +40,8 @@ def main():
 
             e = int(s_e_prod / float(s))
             req = its.objects.manual_capture_request(s, e)
+
+            # TODO: Capture using raw-only, once it works reliably.
             cap,_ = cam.do_capture(req, cam.CAP_RAW_YUV)
 
             # Measure the variance. Each shot should be noisier than the
@@ -56,6 +58,8 @@ def main():
         pylab.plot(range(len(variances)), variances)
         matplotlib.pyplot.savefig("%s_variances.png" % (NAME))
 
+        # TODO: Add pass/fail check for test_raw_sensitivity.
+
 if __name__ == '__main__':
     main()
 
diff --git a/apps/CameraITS/tests/test_sensor_events.py b/apps/CameraITS/tests/scene1/test_sensor_events.py
index 2097310..2097310 100644
--- a/apps/CameraITS/tests/test_sensor_events.py
+++ b/apps/CameraITS/tests/scene1/test_sensor_events.py
diff --git a/apps/CameraITS/tests/test_unified_timestamps.py b/apps/CameraITS/tests/scene1/test_unified_timestamps.py
index d446e02..d446e02 100644
--- a/apps/CameraITS/tests/test_unified_timestamps.py
+++ b/apps/CameraITS/tests/scene1/test_unified_timestamps.py
diff --git a/apps/CameraITS/tests/test_yuv_plus_dng.py b/apps/CameraITS/tests/scene1/test_yuv_plus_dng.py
index 6fff221..6fff221 100644
--- a/apps/CameraITS/tests/test_yuv_plus_dng.py
+++ b/apps/CameraITS/tests/scene1/test_yuv_plus_dng.py
diff --git a/apps/CameraITS/tests/test_yuv_plus_jpeg.py b/apps/CameraITS/tests/scene1/test_yuv_plus_jpeg.py
index 0fa5b77..0fa5b77 100644
--- a/apps/CameraITS/tests/test_yuv_plus_jpeg.py
+++ b/apps/CameraITS/tests/scene1/test_yuv_plus_jpeg.py
diff --git a/apps/CameraITS/tests/test_yuv_plus_raw.py b/apps/CameraITS/tests/scene1/test_yuv_plus_raw.py
index 001af1c..001af1c 100644
--- a/apps/CameraITS/tests/test_yuv_plus_raw.py
+++ b/apps/CameraITS/tests/scene1/test_yuv_plus_raw.py
diff --git a/apps/CameraITS/tests/config.py b/apps/CameraITS/tools/config.py
index 184ef9f..184ef9f 100644
--- a/apps/CameraITS/tests/config.py
+++ b/apps/CameraITS/tools/config.py
diff --git a/apps/CameraITS/tools/run_scene_tests.sh b/apps/CameraITS/tools/run_scene_tests.sh
new file mode 100755
index 0000000..8bae5cb
--- /dev/null
+++ b/apps/CameraITS/tools/run_scene_tests.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Copyright 2014 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -----------------------------------------------------------------------------
+# The tests exercised in this file all assert/exit on failure, and terminate
+# cleanly on success.
+
+# -----------------------------------------------------------------------------
+# Optionally, the device can be rebooted for each test, to ensure that
+# a problem in one test doesn't propagate into subsequent tests; use this
+# when debugging test failures, but not when running the test suite (since
+# all tests should pass even when run back-to-back).
+#REBOOT=reboot
+
+# -----------------------------------------------------------------------------
+# This script should be run from inside a tests/scene<N> directory.
+
+if [ ! -f ../../tools/config.py ]
+then
+    echo "This script must be run from inside a tests/scene<N> directory."
+    exit
+fi
+
+rm -rf out
+mkdir -p out
+
+cd out
+python ../../../tools/config.py $REBOOT
+cd ..
+
+testcount=0
+failcount=0
+
+for T in *.py
+do
+    cd out
+    let testcount=testcount+1
+    echo ""
+    echo "--------------------------------------------------------------------"
+    echo "Running test: $T"
+    echo "--------------------------------------------------------------------"
+    python ../"$T" $REBOOT
+    code=$?
+    if [ $code -ne 0 ]; then
+        let failcount=failcount+1
+        echo ""
+        echo "###############"
+        echo "# Test failed #"
+        echo "###############"
+    fi
+    echo ""
+    cd ..
+done
+
+echo ""
+echo "$failcount out of $testcount tests failed"
+echo ""
+