Fix to AudioLoopbackStress and AdbScreenrecord tests

Fixes to Glitches test; after consultation with Audio Devs, a 4 step approach is taken to run Glitches test to minimize test flakiness.
Also restructures basic test flow - once an error is detected and reported, but rather than returning up the calling hierarchy, an exception is thrown. Before, the test exited incorrectly and reported values even though the test had failed. This change affects both AudioLoopbackStress and AdbScreenrecord.

Bug: 64715929
Bug: 64438393
Change-Id: I518f0c21c00fb1bff0ace17655d907ebdca63207
Fixes:
Test: tradefed.sh run google/test/framework/media/adb-screen-record -s 84B0115625000809
Test:  tradefed.sh  run google/test/framework/media/audio-loopback-stress --iterations 1 --test-type 223 --buffer-test-duration 3600 -s 84B0115625000809
Test:  tradefed.sh  run google/test/framework/media/audio-loopback-stress --iterations 10 -s 84B0115625000809

6 files changed, 214 insertions, 45 deletions

diff --git a/src/com/android/media/tests/AdbScreenrecordTest.java b/src/com/android/media/tests/AdbScreenrecordTest.java
index eb50f50..34ecbde 100644
--- a/src/com/android/media/tests/AdbScreenrecordTest.java
+++ b/src/com/android/media/tests/AdbScreenrecordTest.java
@@ -131,10 +131,13 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest {
 
             // "resultDictionary" can be used to post results to dashboards like BlackBox
             resultsDictionary = runTest(resultsDictionary, TEST_TIMEOUT_MS);
-        } finally {
             final String metricsStr = Arrays.toString(resultsDictionary.entrySet().toArray());
             CLog.i("Uploading metrics values:\n" + metricsStr);
             mTestRunHelper.endTest(resultsDictionary);
+        } catch (TestFailureException e) {
+            CLog.i("TestRunHelper.reportFailure triggered");
+        } finally {
+            deleteFileFromDevice(getAbsoluteFilename());
         }
     }
 
@@ -153,9 +156,10 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest {
      * </ul>
      *
      * @throws DeviceNotAvailableException
+     * @throws TestFailureException
      */
     private Map<String, String> runTest(Map<String, String> results, final long timeout)
-            throws DeviceNotAvailableException {
+            throws DeviceNotAvailableException, TestFailureException {
         final CollectingOutputReceiver receiver = new CollectingOutputReceiver();
         final String cmd = generateAdbScreenRecordCommand();
         final String deviceFileName = getAbsoluteFilename();
@@ -169,15 +173,10 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest {
         CLog.i("Wait for recorded file: " + deviceFileName);
         if (!waitForFile(getDevice(), timeout, deviceFileName)) {
             mTestRunHelper.reportFailure("Recorded test file not found");
-            // Since we don't have a file, no need to delete it; we can return here
-            return results;
         }
 
         CLog.i("Get number of recorded frames and recorded length from adb output");
-        if (!extractVideoDataFromAdbOutput(adbOutput, results)) {
-            deleteFileFromDevice(deviceFileName);
-            return results;
-        }
+        extractVideoDataFromAdbOutput(adbOutput, results);
 
         CLog.i("Get duration and bitrate info from video file using '" + AVPROBE_STR + "'");
         try {
@@ -234,6 +233,10 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest {
      * @throws DeviceNotAvailableException
      */
     private void deleteFileFromDevice(String deviceFileName) throws DeviceNotAvailableException {
+        if (deviceFileName == null || deviceFileName.isEmpty()) {
+            return;
+        }
+
         CLog.i("Delete file from device: " + deviceFileName);
         getDevice().executeShellCommand("rm -f " + deviceFileName);
     }
@@ -243,15 +246,15 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest {
      *
      * @throws DeviceNotAvailableException
      * @throws ParseException
+     * @throws TestFailureException
      */
-    private boolean extractDurationAndBitrateFromVideoFileUsingAvprobe(
+    private void extractDurationAndBitrateFromVideoFileUsingAvprobe(
             String deviceFileName, Map<String, String> results)
-            throws DeviceNotAvailableException, ParseException {
+            throws DeviceNotAvailableException, ParseException, TestFailureException {
         CLog.i("Check if the recorded file has some data in it: " + deviceFileName);
         IFileEntry video = getDevice().getFileEntry(deviceFileName);
         if (video == null || video.getFileEntry().getSizeValue() < 1) {
             mTestRunHelper.reportFailure("Video Entry info failed");
-            return false;
         }
 
         final File recordedVideo = getDevice().pullFile(deviceFileName);
@@ -271,14 +274,12 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest {
 
         if (result.getStatus() != CommandStatus.SUCCESS) {
             mTestRunHelper.reportFailure(AVPROBE_STR + " command failed");
-            return false;
         }
 
         String data = result.getStderr();
         CLog.i("data: " + data);
         if (data == null || data.isEmpty()) {
             mTestRunHelper.reportFailure(AVPROBE_STR + " output data is empty");
-            return false;
         }
 
         Matcher m = Pattern.compile(REGEX_IS_VIDEO_OK).matcher(data);
@@ -286,7 +287,6 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest {
             final String errMsg =
                     "Video verification failed; no matching verification pattern found";
             mTestRunHelper.reportFailure(errMsg);
-            return false;
         }
 
         String duration = m.group(1);
@@ -296,11 +296,12 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest {
 
         results.put(RESULT_KEY_VERIFIED_DURATION, Long.toString(durationInMilliseconds / 1000));
         results.put(RESULT_KEY_VERIFIED_BITRATE, Long.toString(bitrateInKilobits));
-        return true;
     }
 
-    /** Extracts recorded number of frames and recorded video length from adb output */
-    private boolean extractVideoDataFromAdbOutput(String adbOutput, Map<String, String> results) {
+    /** Extracts recorded number of frames and recorded video length from adb output
+     * @throws TestFailureException */
+    private boolean extractVideoDataFromAdbOutput(String adbOutput, Map<String, String> results)
+            throws TestFailureException {
         final String regEx = "recorded (\\d+) frames in (\\d+) second";
         Matcher m = Pattern.compile(regEx).matcher(adbOutput);
         if (!m.find()) {
@@ -387,8 +388,9 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest {
         throw new RuntimeException(err);
     }
 
-    /** Verifies that passed in test parameters are legitimate */
-    private boolean verifyTestParameters() {
+    /** Verifies that passed in test parameters are legitimate
+     * @throws TestFailureException */
+    private boolean verifyTestParameters() throws TestFailureException {
         if (mRecordTimeInSeconds != -1 && mRecordTimeInSeconds < 1) {
             final String error =
                     String.format(ERR_OPTION_MALFORMED, OPTION_TIME_LIMIT, mRecordTimeInSeconds);
diff --git a/src/com/android/media/tests/AudioLevelUtility.java b/src/com/android/media/tests/AudioLevelUtility.java
index 3aee1fb..7898d55 100644
--- a/src/com/android/media/tests/AudioLevelUtility.java
+++ b/src/com/android/media/tests/AudioLevelUtility.java
@@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit;
 /** Class to provide audio level utility functions for a test device */
 public class AudioLevelUtility {
 
-    public static int extractDeviceAudioLevelFromAdbShell(ITestDevice device)
+    public static int extractDeviceHeadsetLevelFromAdbShell(ITestDevice device)
             throws DeviceNotAvailableException {
 
         final String ADB_SHELL_DUMPSYS_AUDIO = "dumpsys audio";
diff --git a/src/com/android/media/tests/AudioLoopbackTest.java b/src/com/android/media/tests/AudioLoopbackTest.java
index 1d9bab5..fea8c40 100644
--- a/src/com/android/media/tests/AudioLoopbackTest.java
+++ b/src/com/android/media/tests/AudioLoopbackTest.java
@@ -19,6 +19,7 @@ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
 
 import com.android.ddmlib.NullOutputReceiver;
 import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.media.tests.AudioLoopbackImageAnalyzer.Result;
 import com.android.media.tests.AudioLoopbackTestHelper.LogFileType;
 import com.android.media.tests.AudioLoopbackTestHelper.ResultData;
 import com.android.tradefed.config.Option;
@@ -156,6 +157,8 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest {
     private static final String KEY_RESULT_PERIOD_CONFIDENCE = "period_confidence";
     private static final String KEY_RESULT_SAMPLING_BLOCK_SIZE = "block_size";
 
+    private static final String REDUCED_GLITCHES_TEST_DURATION = "600"; // 10 min
+
     private static final LogFileType[] LATENCY_TEST_LOGS = {
         LogFileType.RESULT,
         LogFileType.GRAPH,
@@ -322,6 +325,7 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest {
 
         mTestRunHelper.startTest(1);
 
+        Map<String, String> metrics = null;
         try {
             if (!verifyTestParameters()) {
                 return;
@@ -330,28 +334,160 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest {
             // Stop logcat logging so we can record one logcat log per iteration
             getDevice().stopLogcat();
 
-            // Run test iterations
-            for (int i = 0; i < mIterations; i++) {
-                CLog.i("---- Iteration " + i + " of " + (mIterations - 1) + " -----");
-
-                final ResultData d = new ResultData();
-                d.setIteration(i);
-                Map<String, String> resultsDictionary = null;
-                resultsDictionary = runTest(d, getSingleTestTimeoutValue());
-
-                mLoopbackTestHelper.addTestData(d, resultsDictionary);
+            switch (getTestType()) {
+                case GLITCH:
+                    runGlitchesTest(mTestRunHelper, mLoopbackTestHelper);
+                    break;
+                case LATENCY:
+                case LATENCY_STRESS:
+                    // Run test iterations
+                    runLatencyTest(mLoopbackTestHelper, mIterations);
+                    break;
+                default:
+                    break;
             }
 
             mLoopbackTestHelper.processTestData();
-        } finally {
-            Map<String, String> metrics = uploadLogsReturnMetrics(listener);
+            metrics = uploadLogsReturnMetrics(listener);
             CLog.i("Uploading metrics values:\n" + Arrays.toString(metrics.entrySet().toArray()));
             mTestRunHelper.endTest(metrics);
+        } catch (TestFailureException e) {
+            CLog.i("TestRunHelper.reportFailure triggered");
+        } finally {
+            CLog.i("Test ended - cleanup");
             deleteAllTempFiles();
             getDevice().startLogcat();
         }
     }
 
+    private void runLatencyTest(AudioLoopbackTestHelper loopbackTestHelper, int iterations)
+            throws DeviceNotAvailableException, TestFailureException {
+        for (int i = 0; i < iterations; i++) {
+            CLog.i("---- Iteration " + i + " of " + (iterations - 1) + " -----");
+
+            final ResultData d = new ResultData();
+            d.setIteration(i);
+            Map<String, String> resultsDictionary = null;
+            resultsDictionary = runTest(d, getSingleTestTimeoutValue());
+            loopbackTestHelper.addTestData(d, resultsDictionary, true);
+        }
+    }
+
+    /**
+     * Glitches test, strategy:
+     * <p>
+     *
+     * <ul>
+     *   <li>1. Calibrate Audio level
+     *   <li>2. Run Audio Latency test until seeing good waveform
+     *   <li>3. Run small Glitches test, 5-10 seconds
+     *   <li>4. If numbers look good, run long Glitches test, else run reduced Glitches test
+     * </ul>
+     *
+     * @param testRunHelper
+     * @param loopbackTestHelper
+     * @throws DeviceNotAvailableException
+     * @throws TestFailureException
+     */
+    private void runGlitchesTest(TestRunHelper testRunHelper,
+            AudioLoopbackTestHelper loopbackTestHelper)
+                    throws DeviceNotAvailableException, TestFailureException {
+        final int MAX_RETRIES = 3;
+        int nrOfSuccessfulTests;
+        int counter = 0;
+        AudioLoopbackTestHelper tempTestHelper = null;
+        boolean runningReducedGlitchesTest = false;
+
+        // Step 1: Calibrate Audio level
+        // Step 2: Run Audio Latency test until seeing good waveform
+        final int LOOPBACK_ITERATIONS = 4;
+        final String originalTestType = mTestType;
+        final String originalBufferTestDuration = mBufferTestDuration;
+        mTestType = TESTTYPE_LATENCY_STR;
+        do {
+            nrOfSuccessfulTests = 0;
+            tempTestHelper = new AudioLoopbackTestHelper(LOOPBACK_ITERATIONS);
+            runLatencyTest(tempTestHelper, LOOPBACK_ITERATIONS);
+            nrOfSuccessfulTests = tempTestHelper.processTestData();
+            counter++;
+        } while (nrOfSuccessfulTests <= 0 && counter <= MAX_RETRIES);
+
+        if (nrOfSuccessfulTests <= 0) {
+            testRunHelper.reportFailure("Glitch Setup failed: Latency test");
+        }
+
+        // Retrieve audio level from successful test
+        int audioLevel = -1;
+        List<ResultData> results = tempTestHelper.getAllTestData();
+        for (ResultData rd : results) {
+            // Check if test passed
+            if (rd.getImageAnalyzerResult() == Result.PASS && rd.getConfidence() == 1.0) {
+                audioLevel = rd.getAudioLevel();
+                break;
+            }
+        }
+
+        if (audioLevel < 6) {
+            testRunHelper.reportFailure("Glitch Setup failed: Audio level not valid");
+        }
+
+        CLog.i("Audio Glitch: Audio level is " + audioLevel);
+
+        // Step 3: Run small Glitches test, 5-10 seconds
+        mTestType = originalTestType;
+        mBufferTestDuration = "10";
+        mAudioLevel = Integer.toString(audioLevel);
+
+        counter = 0;
+        int glitches = -1;
+        do {
+            tempTestHelper = new AudioLoopbackTestHelper(1);
+            runLatencyTest(tempTestHelper, 1);
+            Map<String, String> resultsDictionary =
+                    tempTestHelper.getResultDictionaryForIteration(0);
+            final String nrOfGlitches =
+                    resultsDictionary.get(getMetricsKey(KEY_RESULT_NUMBER_OF_GLITCHES));
+            glitches = Integer.parseInt(nrOfGlitches);
+            CLog.i("10 s glitch test produced " + glitches + " glitches");
+            counter++;
+        } while (glitches > 10 || glitches < 0 && counter <= MAX_RETRIES);
+
+        // Step 4: If numbers look good, run long Glitches test
+        if (glitches > 10 || glitches < 0) {
+            // Reduce test time and set some values to 0 once test completes
+            runningReducedGlitchesTest = true;
+            mBufferTestDuration = REDUCED_GLITCHES_TEST_DURATION;
+        } else {
+            mBufferTestDuration = originalBufferTestDuration;
+        }
+
+        final ResultData d = new ResultData();
+        d.setIteration(0);
+        Map<String, String> resultsDictionary = null;
+        resultsDictionary = runTest(d, getSingleTestTimeoutValue());
+        if (runningReducedGlitchesTest) {
+            // Special treatment, we want to upload values, but also indicate that pre-test
+            // conditions failed. We will set the glitches count and zero out the rest.
+            String[] testValuesToChangeArray = new String[] {
+                KEY_RESULT_RECORDER_BENCHMARK,
+                KEY_RESULT_RECORDER_OUTLIER,
+                KEY_RESULT_PLAYER_BENCHMARK,
+                KEY_RESULT_PLAYER_OUTLIER,
+                KEY_RESULT_RECORDER_BUFFER_CALLBACK,
+                KEY_RESULT_PLAYER_BUFFER_CALLBACK
+            };
+
+            for (String key : testValuesToChangeArray) {
+                final String metricsKey = getMetricsKey(key);
+                if (resultsDictionary.containsKey(metricsKey)) {
+                    resultsDictionary.put(metricsKey, "0");
+                }
+            }
+        }
+
+        loopbackTestHelper.addTestData(d, resultsDictionary, false);
+    }
+
     private void initializeTest(ITestInvocationListener listener)
             throws UnsupportedOperationException, DeviceNotAvailableException {
 
@@ -370,7 +506,7 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest {
     }
 
     private Map<String, String> runTest(ResultData data, final long timeout)
-            throws DeviceNotAvailableException {
+            throws DeviceNotAvailableException, TestFailureException {
 
         // start measurement and wait for result file
         final NullOutputReceiver receiver = new NullOutputReceiver();
@@ -434,7 +570,7 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest {
 
             // Trust but verify, so get Audio Level from ADB and compare to value from app
             final int adbAudioLevel =
-                    AudioLevelUtility.extractDeviceAudioLevelFromAdbShell(getDevice());
+                    AudioLevelUtility.extractDeviceHeadsetLevelFromAdbShell(getDevice());
             if (data.getAudioLevel() != adbAudioLevel) {
                 final String errMsg =
                         String.format(
@@ -461,7 +597,7 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest {
     }
 
     private Map<String, String> uploadLogsReturnMetrics(ITestInvocationListener listener)
-            throws DeviceNotAvailableException {
+            throws DeviceNotAvailableException, TestFailureException {
 
         // "resultDictionary" is used to post results to dashboards like BlackBox
         // "results" contains test logs to be uploaded; i.e. to Sponge
@@ -545,7 +681,7 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest {
         return TestType.NONE;
     }
 
-    private boolean verifyTestParameters() {
+    private boolean verifyTestParameters() throws TestFailureException {
         if (getTestType() != TestType.NONE) {
             return true;
         }
diff --git a/src/com/android/media/tests/AudioLoopbackTestHelper.java b/src/com/android/media/tests/AudioLoopbackTestHelper.java
index 4e9c2b0..c9e058a 100644
--- a/src/com/android/media/tests/AudioLoopbackTestHelper.java
+++ b/src/com/android/media/tests/AudioLoopbackTestHelper.java
@@ -287,15 +287,20 @@ public class AudioLoopbackTestHelper {
         mAllResults = new ArrayList<ResultData>(iterations);
     }
 
-    public void addTestData(ResultData data, Map<String, String> resultDictionary) {
+    public void addTestData(ResultData data,
+            Map<String,
+            String> resultDictionary,
+            boolean useImageAnalyzer) {
         mResultDictionaries.add(data.getIteration(), resultDictionary);
         mAllResults.add(data);
 
-        // Analyze captured screenshot to see if wave form is within reason
-        final String screenshot = data.getLogFile(LogFileType.GRAPH);
-        final Pair<Result, String> result = AudioLoopbackImageAnalyzer.analyzeImage(screenshot);
-        data.setImageAnalyzerResult(result.first);
-        data.setFailureReason(result.second);
+        if (useImageAnalyzer) {
+            // Analyze captured screenshot to see if wave form is within reason
+            final String screenshot = data.getLogFile(LogFileType.GRAPH);
+            final Pair<Result, String> result = AudioLoopbackImageAnalyzer.analyzeImage(screenshot);
+            data.setImageAnalyzerResult(result.first);
+            data.setFailureReason(result.second);
+        }
     }
 
     public final List<ResultData> getAllTestData() {
diff --git a/src/com/android/media/tests/TestFailureException.java b/src/com/android/media/tests/TestFailureException.java
new file mode 100644
index 0000000..c33c195
--- /dev/null
+++ b/src/com/android/media/tests/TestFailureException.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.media.tests;
+
+/** Exception used to indicate test failure. */
+public class TestFailureException extends RuntimeException {
+   static final long serialVersionUID = 1L;
+
+    public TestFailureException() {
+        super();
+    }
+}
diff --git a/src/com/android/media/tests/TestRunHelper.java b/src/com/android/media/tests/TestRunHelper.java
index f752cf3..f619772 100644
--- a/src/com/android/media/tests/TestRunHelper.java
+++ b/src/com/android/media/tests/TestRunHelper.java
@@ -39,18 +39,19 @@ public class TestRunHelper {
         return mTestStopTime - mTestStartTime;
     }
 
-    public void reportFailure(String errMsg) {
+    public void reportFailure(String errMsg) throws TestFailureException {
         CLog.e(errMsg);
+        mListener.testRunFailed(errMsg);
         mListener.testFailed(mTestId, errMsg);
         mListener.testEnded(mTestId, new HashMap<String, String>());
-        mListener.testRunFailed(errMsg);
+        throw new TestFailureException();
     }
 
     /** @param resultDictionary */
     public void endTest(Map<String, String> resultDictionary) {
         mTestStopTime = System.currentTimeMillis();
-        mListener.testEnded(mTestId, resultDictionary);
         mListener.testRunEnded(getTotalTestTime(), resultDictionary);
+        mListener.testEnded(mTestId, resultDictionary);
     }
 
     public void startTest(int numberOfTests) {