CameraITS: Improved pass/fail checking on some tests

Change-Id: I8c543291d1a71adfb39e41561339f92e1c80a2da

9 files changed, 99 insertions, 17 deletions

diff --git a/apps/CameraITS/tests/regress.sh b/apps/CameraITS/tests/regress.sh
index 614d2ae..080df3f 100755
--- a/apps/CameraITS/tests/regress.sh
+++ b/apps/CameraITS/tests/regress.sh
@@ -38,26 +38,28 @@ for T in \
          test_capture_result.py \
          test_exposure.py \
          test_formats.py \
+         test_jitter.py \
          test_jpeg.py \
          test_param_color_correction.py \
          test_param_exposure_time.py \
          test_param_noise_reduction.py \
          test_param_sensitivity.py \
+         test_param_sensitivity_burst.py \
          test_param_tonemap_mode.py \
+         test_yuv_plus_dng.py \
+         test_yuv_plus_jpeg.py \
+         test_yuv_plus_raw.py \
          \
          test_auto.py \
          test_crop_region.py \
-         test_jitter.py \
+         test_format_combos.py \
          test_latching.py \
          test_linearity.py \
          test_long_burst.py \
          test_param_edge_mode.py \
          test_param_flash_mode.py \
          test_test_patterns.py \
-         test_yuv_plus_jpeg.py \
-         test_yuv_plus_raw.py \
 
-         #test_format_combos.py \
          #test_blc_lsc.py \
          #test_dng_tags.py \
 
diff --git a/apps/CameraITS/tests/test_formats.py b/apps/CameraITS/tests/test_formats.py
index 67b3f8b..16fc3fd 100644
--- a/apps/CameraITS/tests/test_formats.py
+++ b/apps/CameraITS/tests/test_formats.py
@@ -15,26 +15,42 @@
 import its.image
 import its.device
 import its.objects
+import its.target
 import os.path
 import Image
+import math
 
 def main():
     """Test that the reported sizes and formats for image capture work.
     """
     NAME = os.path.basename(__file__).split(".")[0]
 
+    # TODO: Look into why some of the RMS diffs are higher in this test.
+    THRESHOLD_MAX_RMS_DIFF = 0.1
+
     with its.device.ItsSession() as cam:
         props = cam.get_camera_properties()
+
+        # Use a manual request with a linear tonemap so that the YUV and JPEG
+        # should look the same (once converted by the its.image module).
+        e, s = its.target.get_target_exposure_combos(cam)["midExposureTime"]
+        req = its.objects.manual_capture_request(s, e, True)
+
+        rgbs = []
+
         for size in its.objects.get_available_output_sizes("yuv", props):
-            req = its.objects.manual_capture_request(100,10*1000*1000)
             out_surface = {"width":size[0], "height":size[1], "format":"yuv"}
             cap = cam.do_capture(req, out_surface)
             assert(cap["format"] == "yuv")
             assert(cap["width"] == size[0])
             assert(cap["height"] == size[1])
             print "Captured YUV %dx%d" % (cap["width"], cap["height"])
+            img = its.image.convert_capture_to_rgb_image(cap)
+            tile = its.image.get_image_patch(img, 0.45, 0.45, 0.1, 0.1)
+            rgb = its.image.compute_image_means(tile)
+            rgbs.append(rgb)
+
         for size in its.objects.get_available_output_sizes("jpg", props):
-            req = its.objects.manual_capture_request(100,10*1000*1000)
             out_surface = {"width":size[0], "height":size[1], "format":"jpg"}
             cap = cam.do_capture(req, out_surface)
             assert(cap["format"] == "jpeg")
@@ -45,6 +61,18 @@ def main():
             assert(img.shape[1] == size[0])
             assert(img.shape[2] == 3)
             print "Captured JPEG %dx%d" % (cap["width"], cap["height"])
+            tile = its.image.get_image_patch(img, 0.45, 0.45, 0.1, 0.1)
+            rgb = its.image.compute_image_means(tile)
+            rgbs.append(rgb)
+
+        max_diff = 0
+        rgb0 = rgbs[0]
+        for rgb1 in rgbs[1:]:
+            rms_diff = math.sqrt(
+                    sum([pow(rgb0[i] - rgb1[i], 2.0) for i in range(3)]) / 3.0)
+            max_diff = max(max_diff, rms_diff)
+        print "Max RMS difference:", max_diff
+        assert(rms_diff < THRESHOLD_MAX_RMS_DIFF)
 
 if __name__ == '__main__':
     main()
diff --git a/apps/CameraITS/tests/test_jitter.py b/apps/CameraITS/tests/test_jitter.py
index 18a1a66..9fb20d1 100644
--- a/apps/CameraITS/tests/test_jitter.py
+++ b/apps/CameraITS/tests/test_jitter.py
@@ -25,6 +25,11 @@ def main():
     """
     NAME = os.path.basename(__file__).split(".")[0]
 
+    # Pass/fail thresholds
+    MIN_AVG_FRAME_DELTA = 30 # at least 30ms delta between frames
+    MAX_VAR_FRAME_DELTA = 0.01 # variance of frame deltas
+    MAX_FRAME_DELTA_JITTER = 0.3 # max ms gap from the average frame delta
+
     with its.device.ItsSession() as cam:
         # ISO 100, 1ms manual exposure, VGA resolution YUV frames.
         req = its.objects.manual_capture_request(100, 1*1000*1000)
@@ -38,14 +43,22 @@ def main():
         deltas_ms = [d/1000000.0 for d in deltas]
         avg = sum(deltas_ms) / len(deltas_ms)
         var = sum([d*d for d in deltas_ms]) / len(deltas_ms) - avg * avg
+        range0 = min(deltas_ms) - avg
+        range1 = max(deltas_ms) - avg
         print "Average:", avg
         print "Variance:", var
-        print "Jitter range:", min(deltas_ms) - avg, "to", max(deltas_ms) - avg
+        print "Jitter range:", range0, "to", range1
 
         # Draw a plot.
         pylab.plot(range(len(deltas_ms)), deltas_ms)
         matplotlib.pyplot.savefig("%s_deltas.png" % (NAME))
 
+        # Test for pass/fail.
+        assert(avg > MIN_AVG_FRAME_DELTA)
+        assert(var < MAX_VAR_FRAME_DELTA)
+        assert(abs(range0) < MAX_FRAME_DELTA_JITTER)
+        assert(abs(range1) < MAX_FRAME_DELTA_JITTER)
+
 if __name__ == '__main__':
     main()
 
diff --git a/apps/CameraITS/tests/test_jpeg.py b/apps/CameraITS/tests/test_jpeg.py
index 8cd2bc1..54a6189 100644
--- a/apps/CameraITS/tests/test_jpeg.py
+++ b/apps/CameraITS/tests/test_jpeg.py
@@ -15,6 +15,7 @@
 import its.image
 import its.device
 import its.objects
+import its.target
 import os.path
 import Image
 import shutil
@@ -26,13 +27,15 @@ def main():
     """
     NAME = os.path.basename(__file__).split(".")[0]
 
-    THRESHOLD_MAX_RMS_DIFF = 0.1
+    THRESHOLD_MAX_RMS_DIFF = 0.01
 
     with its.device.ItsSession() as cam:
         props = cam.get_camera_properties()
 
+        e, s = its.target.get_target_exposure_combos(cam)["midExposureTime"]
+        req = its.objects.manual_capture_request(s, e, True)
+
         # YUV
-        req = its.objects.manual_capture_request(100,100*1000*1000)
         size = its.objects.get_available_output_sizes("yuv", props)[0]
         out_surface = {"width":size[0], "height":size[1], "format":"yuv"}
         cap = cam.do_capture(req, out_surface)
@@ -42,7 +45,6 @@ def main():
         rgb0 = its.image.compute_image_means(tile)
 
         # JPEG
-        req = its.objects.manual_capture_request(100,100*1000*1000)
         size = its.objects.get_available_output_sizes("jpg", props)[0]
         out_surface = {"width":size[0], "height":size[1], "format":"jpg"}
         cap = cam.do_capture(req, out_surface)
diff --git a/apps/CameraITS/tests/test_param_sensitivity_burst.py b/apps/CameraITS/tests/test_param_sensitivity_burst.py
index 315aee1..2df1d08 100644
--- a/apps/CameraITS/tests/test_param_sensitivity_burst.py
+++ b/apps/CameraITS/tests/test_param_sensitivity_burst.py
@@ -35,7 +35,7 @@ def main():
         for i,cap in enumerate(caps):
             s_req = sensitivities[i]
             s_res = cap["metadata"]["android.sensor.sensitivity"]
-            print s_req, s_res
+            assert(s_req == s_res)
 
 if __name__ == '__main__':
     main()
diff --git a/apps/CameraITS/tests/test_param_tonemap_mode.py b/apps/CameraITS/tests/test_param_tonemap_mode.py
index 2921e23..919d279 100644
--- a/apps/CameraITS/tests/test_param_tonemap_mode.py
+++ b/apps/CameraITS/tests/test_param_tonemap_mode.py
@@ -15,6 +15,7 @@
 import its.image
 import its.device
 import its.objects
+import its.target
 import os
 import sys
 import numpy
@@ -42,6 +43,9 @@ def main():
 
     with its.device.ItsSession() as cam:
 
+        e, s = its.target.get_target_exposure_combos(cam)["midExposureTime"]
+        e /= 2
+
         # Test 1: that the tonemap curves have the expected effect. Take two
         # shots, with n in [0,1], where each has a linear tonemap, with the
         # n=1 shot having a steeper gradient. The gradient for each R,G,B
@@ -50,7 +54,7 @@ def main():
         rgb_means = []
 
         for n in [0,1]:
-            req = its.objects.manual_capture_request(100,50*1000*1000)
+            req = its.objects.manual_capture_request(s,e)
             req["android.tonemap.mode"] = 0
             req["android.tonemap.curveRed"] = (
                     sum([[i/LM1, min(1.0,(1+0.5*n)*i/LM1)] for i in range(L)], []))
@@ -78,7 +82,7 @@ def main():
         for size in [32,64]:
             m = float(size-1)
             curve = sum([[i/m, i/m] for i in range(size)], [])
-            req = its.objects.manual_capture_request(100,50*1000*1000)
+            req = its.objects.manual_capture_request(s,e)
             req["android.tonemap.mode"] = 0
             req["android.tonemap.curveRed"] = curve
             req["android.tonemap.curveGreen"] = curve
diff --git a/apps/CameraITS/tests/test_yuv_plus_dng.py b/apps/CameraITS/tests/test_yuv_plus_dng.py
index e15a789..6fff221 100644
--- a/apps/CameraITS/tests/test_yuv_plus_dng.py
+++ b/apps/CameraITS/tests/test_yuv_plus_dng.py
@@ -36,6 +36,9 @@ def main():
         with open("%s.dng"%(NAME), "wb") as f:
             f.write(cap_dng["data"])
 
+        # No specific pass/fail check; test is assumed to have succeeded if
+        # it completes.
+
 if __name__ == '__main__':
     main()
 
diff --git a/apps/CameraITS/tests/test_yuv_plus_jpeg.py b/apps/CameraITS/tests/test_yuv_plus_jpeg.py
index 10b0635..0fa5b77 100644
--- a/apps/CameraITS/tests/test_yuv_plus_jpeg.py
+++ b/apps/CameraITS/tests/test_yuv_plus_jpeg.py
@@ -15,14 +15,16 @@
 import its.image
 import its.device
 import its.objects
+import its.target
 import os.path
+import math
 
 def main():
     """Test capturing a single frame as both YUV and JPEG outputs.
     """
     NAME = os.path.basename(__file__).split(".")[0]
 
-    req = its.objects.auto_capture_request()
+    THRESHOLD_MAX_RMS_DIFF = 0.01
 
     # Hard-code a preview (VGA) size for the YUV image.
     # TODO: Replace this with code to select sizes from what's available.
@@ -30,14 +32,27 @@ def main():
     fmt_jpeg = {"format":"jpeg"}
 
     with its.device.ItsSession() as cam:
-        cam.do_3a();
+        # Use a manual request with a linear tonemap so that the YUV and JPEG
+        # should look the same (once converted by the its.image module).
+        e, s = its.target.get_target_exposure_combos(cam)["midExposureTime"]
+        req = its.objects.manual_capture_request(s, e, True)
+
         cap_yuv, cap_jpeg = cam.do_capture(req, [fmt_yuv, fmt_jpeg])
 
         img = its.image.convert_capture_to_rgb_image(cap_yuv)
         its.image.write_image(img, "%s_yuv.jpg" % (NAME))
+        tile = its.image.get_image_patch(img, 0.45, 0.45, 0.1, 0.1)
+        rgb0 = its.image.compute_image_means(tile)
 
         img = its.image.convert_capture_to_rgb_image(cap_jpeg)
         its.image.write_image(img, "%s_jpeg.jpg" % (NAME))
+        tile = its.image.get_image_patch(img, 0.45, 0.45, 0.1, 0.1)
+        rgb1 = its.image.compute_image_means(tile)
+
+        rms_diff = math.sqrt(
+                sum([pow(rgb0[i] - rgb1[i], 2.0) for i in range(3)]) / 3.0)
+        print "RMS difference:", rms_diff
+        assert(rms_diff < THRESHOLD_MAX_RMS_DIFF)
 
 if __name__ == '__main__':
     main()
diff --git a/apps/CameraITS/tests/test_yuv_plus_raw.py b/apps/CameraITS/tests/test_yuv_plus_raw.py
index 157b924..504af3a 100644
--- a/apps/CameraITS/tests/test_yuv_plus_raw.py
+++ b/apps/CameraITS/tests/test_yuv_plus_raw.py
@@ -15,26 +15,41 @@
 import its.image
 import its.device
 import its.objects
+import its.target
 import os.path
+import math
 
 def main():
     """Test capturing a single frame as both RAW and YUV outputs.
     """
     NAME = os.path.basename(__file__).split(".")[0]
 
+    THRESHOLD_MAX_RMS_DIFF = 0.01
+
     with its.device.ItsSession() as cam:
         props = cam.get_camera_properties()
 
-        cam.do_3a()
+        # Use a manual request with a linear tonemap so that the YUV and RAW
+        # should look the same (once converted by the its.image module).
+        e, s = its.target.get_target_exposure_combos(cam)["midExposureTime"]
+        req = its.objects.manual_capture_request(s, e, True)
 
-        req = its.objects.auto_capture_request()
         cap_raw, cap_yuv = cam.do_capture(req, cam.CAP_RAW_YUV)
 
         img = its.image.convert_capture_to_rgb_image(cap_yuv)
         its.image.write_image(img, "%s_yuv.jpg" % (NAME))
+        tile = its.image.get_image_patch(img, 0.45, 0.45, 0.1, 0.1)
+        rgb0 = its.image.compute_image_means(tile)
 
         img = its.image.convert_capture_to_rgb_image(cap_raw, props=props)
         its.image.write_image(img, "%s_raw.jpg" % (NAME), True)
+        tile = its.image.get_image_patch(img, 0.45, 0.45, 0.1, 0.1)
+        rgb1 = its.image.compute_image_means(tile)
+
+        rms_diff = math.sqrt(
+                sum([pow(rgb0[i] - rgb1[i], 2.0) for i in range(3)]) / 3.0)
+        print "RMS difference:", rms_diff
+        assert(rms_diff < THRESHOLD_MAX_RMS_DIFF)
 
 if __name__ == '__main__':
     main()