diff options
Diffstat (limited to 'src')
6 files changed, 214 insertions, 45 deletions
diff --git a/src/com/android/media/tests/AdbScreenrecordTest.java b/src/com/android/media/tests/AdbScreenrecordTest.java index eb50f50..34ecbde 100644 --- a/src/com/android/media/tests/AdbScreenrecordTest.java +++ b/src/com/android/media/tests/AdbScreenrecordTest.java @@ -131,10 +131,13 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest { // "resultDictionary" can be used to post results to dashboards like BlackBox resultsDictionary = runTest(resultsDictionary, TEST_TIMEOUT_MS); - } finally { final String metricsStr = Arrays.toString(resultsDictionary.entrySet().toArray()); CLog.i("Uploading metrics values:\n" + metricsStr); mTestRunHelper.endTest(resultsDictionary); + } catch (TestFailureException e) { + CLog.i("TestRunHelper.reportFailure triggered"); + } finally { + deleteFileFromDevice(getAbsoluteFilename()); } } @@ -153,9 +156,10 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest { * </ul> * * @throws DeviceNotAvailableException + * @throws TestFailureException */ private Map<String, String> runTest(Map<String, String> results, final long timeout) - throws DeviceNotAvailableException { + throws DeviceNotAvailableException, TestFailureException { final CollectingOutputReceiver receiver = new CollectingOutputReceiver(); final String cmd = generateAdbScreenRecordCommand(); final String deviceFileName = getAbsoluteFilename(); @@ -169,15 +173,10 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest { CLog.i("Wait for recorded file: " + deviceFileName); if (!waitForFile(getDevice(), timeout, deviceFileName)) { mTestRunHelper.reportFailure("Recorded test file not found"); - // Since we don't have a file, no need to delete it; we can return here - return results; } CLog.i("Get number of recorded frames and recorded length from adb output"); - if (!extractVideoDataFromAdbOutput(adbOutput, results)) { - deleteFileFromDevice(deviceFileName); - return results; - } + extractVideoDataFromAdbOutput(adbOutput, results); CLog.i("Get duration and bitrate info from video file using '" + AVPROBE_STR + "'"); try { @@ -234,6 +233,10 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest { * @throws DeviceNotAvailableException */ private void deleteFileFromDevice(String deviceFileName) throws DeviceNotAvailableException { + if (deviceFileName == null || deviceFileName.isEmpty()) { + return; + } + CLog.i("Delete file from device: " + deviceFileName); getDevice().executeShellCommand("rm -f " + deviceFileName); } @@ -243,15 +246,15 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest { * * @throws DeviceNotAvailableException * @throws ParseException + * @throws TestFailureException */ - private boolean extractDurationAndBitrateFromVideoFileUsingAvprobe( + private void extractDurationAndBitrateFromVideoFileUsingAvprobe( String deviceFileName, Map<String, String> results) - throws DeviceNotAvailableException, ParseException { + throws DeviceNotAvailableException, ParseException, TestFailureException { CLog.i("Check if the recorded file has some data in it: " + deviceFileName); IFileEntry video = getDevice().getFileEntry(deviceFileName); if (video == null || video.getFileEntry().getSizeValue() < 1) { mTestRunHelper.reportFailure("Video Entry info failed"); - return false; } final File recordedVideo = getDevice().pullFile(deviceFileName); @@ -271,14 +274,12 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest { if (result.getStatus() != CommandStatus.SUCCESS) { mTestRunHelper.reportFailure(AVPROBE_STR + " command failed"); - return false; } String data = result.getStderr(); CLog.i("data: " + data); if (data == null || data.isEmpty()) { mTestRunHelper.reportFailure(AVPROBE_STR + " output data is empty"); - return false; } Matcher m = Pattern.compile(REGEX_IS_VIDEO_OK).matcher(data); @@ -286,7 +287,6 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest { final String errMsg = "Video verification failed; no matching verification pattern found"; mTestRunHelper.reportFailure(errMsg); - return false; } String duration = m.group(1); @@ -296,11 +296,12 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest { results.put(RESULT_KEY_VERIFIED_DURATION, Long.toString(durationInMilliseconds / 1000)); results.put(RESULT_KEY_VERIFIED_BITRATE, Long.toString(bitrateInKilobits)); - return true; } - /** Extracts recorded number of frames and recorded video length from adb output */ - private boolean extractVideoDataFromAdbOutput(String adbOutput, Map<String, String> results) { + /** Extracts recorded number of frames and recorded video length from adb output + * @throws TestFailureException */ + private boolean extractVideoDataFromAdbOutput(String adbOutput, Map<String, String> results) + throws TestFailureException { final String regEx = "recorded (\\d+) frames in (\\d+) second"; Matcher m = Pattern.compile(regEx).matcher(adbOutput); if (!m.find()) { @@ -387,8 +388,9 @@ public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest { throw new RuntimeException(err); } - /** Verifies that passed in test parameters are legitimate */ - private boolean verifyTestParameters() { + /** Verifies that passed in test parameters are legitimate + * @throws TestFailureException */ + private boolean verifyTestParameters() throws TestFailureException { if (mRecordTimeInSeconds != -1 && mRecordTimeInSeconds < 1) { final String error = String.format(ERR_OPTION_MALFORMED, OPTION_TIME_LIMIT, mRecordTimeInSeconds); diff --git a/src/com/android/media/tests/AudioLevelUtility.java b/src/com/android/media/tests/AudioLevelUtility.java index 3aee1fb..7898d55 100644 --- a/src/com/android/media/tests/AudioLevelUtility.java +++ b/src/com/android/media/tests/AudioLevelUtility.java @@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit; /** Class to provide audio level utility functions for a test device */ public class AudioLevelUtility { - public static int extractDeviceAudioLevelFromAdbShell(ITestDevice device) + public static int extractDeviceHeadsetLevelFromAdbShell(ITestDevice device) throws DeviceNotAvailableException { final String ADB_SHELL_DUMPSYS_AUDIO = "dumpsys audio"; diff --git a/src/com/android/media/tests/AudioLoopbackTest.java b/src/com/android/media/tests/AudioLoopbackTest.java index 1d9bab5..fea8c40 100644 --- a/src/com/android/media/tests/AudioLoopbackTest.java +++ b/src/com/android/media/tests/AudioLoopbackTest.java @@ -19,6 +19,7 @@ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import com.android.ddmlib.NullOutputReceiver; import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.media.tests.AudioLoopbackImageAnalyzer.Result; import com.android.media.tests.AudioLoopbackTestHelper.LogFileType; import com.android.media.tests.AudioLoopbackTestHelper.ResultData; import com.android.tradefed.config.Option; @@ -156,6 +157,8 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { private static final String KEY_RESULT_PERIOD_CONFIDENCE = "period_confidence"; private static final String KEY_RESULT_SAMPLING_BLOCK_SIZE = "block_size"; + private static final String REDUCED_GLITCHES_TEST_DURATION = "600"; // 10 min + private static final LogFileType[] LATENCY_TEST_LOGS = { LogFileType.RESULT, LogFileType.GRAPH, @@ -322,6 +325,7 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { mTestRunHelper.startTest(1); + Map<String, String> metrics = null; try { if (!verifyTestParameters()) { return; @@ -330,28 +334,160 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { // Stop logcat logging so we can record one logcat log per iteration getDevice().stopLogcat(); - // Run test iterations - for (int i = 0; i < mIterations; i++) { - CLog.i("---- Iteration " + i + " of " + (mIterations - 1) + " -----"); - - final ResultData d = new ResultData(); - d.setIteration(i); - Map<String, String> resultsDictionary = null; - resultsDictionary = runTest(d, getSingleTestTimeoutValue()); - - mLoopbackTestHelper.addTestData(d, resultsDictionary); + switch (getTestType()) { + case GLITCH: + runGlitchesTest(mTestRunHelper, mLoopbackTestHelper); + break; + case LATENCY: + case LATENCY_STRESS: + // Run test iterations + runLatencyTest(mLoopbackTestHelper, mIterations); + break; + default: + break; } mLoopbackTestHelper.processTestData(); - } finally { - Map<String, String> metrics = uploadLogsReturnMetrics(listener); + metrics = uploadLogsReturnMetrics(listener); CLog.i("Uploading metrics values:\n" + Arrays.toString(metrics.entrySet().toArray())); mTestRunHelper.endTest(metrics); + } catch (TestFailureException e) { + CLog.i("TestRunHelper.reportFailure triggered"); + } finally { + CLog.i("Test ended - cleanup"); deleteAllTempFiles(); getDevice().startLogcat(); } } + private void runLatencyTest(AudioLoopbackTestHelper loopbackTestHelper, int iterations) + throws DeviceNotAvailableException, TestFailureException { + for (int i = 0; i < iterations; i++) { + CLog.i("---- Iteration " + i + " of " + (iterations - 1) + " -----"); + + final ResultData d = new ResultData(); + d.setIteration(i); + Map<String, String> resultsDictionary = null; + resultsDictionary = runTest(d, getSingleTestTimeoutValue()); + loopbackTestHelper.addTestData(d, resultsDictionary, true); + } + } + + /** + * Glitches test, strategy: + * <p> + * + * <ul> + * <li>1. Calibrate Audio level + * <li>2. Run Audio Latency test until seeing good waveform + * <li>3. Run small Glitches test, 5-10 seconds + * <li>4. If numbers look good, run long Glitches test, else run reduced Glitches test + * </ul> + * + * @param testRunHelper + * @param loopbackTestHelper + * @throws DeviceNotAvailableException + * @throws TestFailureException + */ + private void runGlitchesTest(TestRunHelper testRunHelper, + AudioLoopbackTestHelper loopbackTestHelper) + throws DeviceNotAvailableException, TestFailureException { + final int MAX_RETRIES = 3; + int nrOfSuccessfulTests; + int counter = 0; + AudioLoopbackTestHelper tempTestHelper = null; + boolean runningReducedGlitchesTest = false; + + // Step 1: Calibrate Audio level + // Step 2: Run Audio Latency test until seeing good waveform + final int LOOPBACK_ITERATIONS = 4; + final String originalTestType = mTestType; + final String originalBufferTestDuration = mBufferTestDuration; + mTestType = TESTTYPE_LATENCY_STR; + do { + nrOfSuccessfulTests = 0; + tempTestHelper = new AudioLoopbackTestHelper(LOOPBACK_ITERATIONS); + runLatencyTest(tempTestHelper, LOOPBACK_ITERATIONS); + nrOfSuccessfulTests = tempTestHelper.processTestData(); + counter++; + } while (nrOfSuccessfulTests <= 0 && counter <= MAX_RETRIES); + + if (nrOfSuccessfulTests <= 0) { + testRunHelper.reportFailure("Glitch Setup failed: Latency test"); + } + + // Retrieve audio level from successful test + int audioLevel = -1; + List<ResultData> results = tempTestHelper.getAllTestData(); + for (ResultData rd : results) { + // Check if test passed + if (rd.getImageAnalyzerResult() == Result.PASS && rd.getConfidence() == 1.0) { + audioLevel = rd.getAudioLevel(); + break; + } + } + + if (audioLevel < 6) { + testRunHelper.reportFailure("Glitch Setup failed: Audio level not valid"); + } + + CLog.i("Audio Glitch: Audio level is " + audioLevel); + + // Step 3: Run small Glitches test, 5-10 seconds + mTestType = originalTestType; + mBufferTestDuration = "10"; + mAudioLevel = Integer.toString(audioLevel); + + counter = 0; + int glitches = -1; + do { + tempTestHelper = new AudioLoopbackTestHelper(1); + runLatencyTest(tempTestHelper, 1); + Map<String, String> resultsDictionary = + tempTestHelper.getResultDictionaryForIteration(0); + final String nrOfGlitches = + resultsDictionary.get(getMetricsKey(KEY_RESULT_NUMBER_OF_GLITCHES)); + glitches = Integer.parseInt(nrOfGlitches); + CLog.i("10 s glitch test produced " + glitches + " glitches"); + counter++; + } while (glitches > 10 || glitches < 0 && counter <= MAX_RETRIES); + + // Step 4: If numbers look good, run long Glitches test + if (glitches > 10 || glitches < 0) { + // Reduce test time and set some values to 0 once test completes + runningReducedGlitchesTest = true; + mBufferTestDuration = REDUCED_GLITCHES_TEST_DURATION; + } else { + mBufferTestDuration = originalBufferTestDuration; + } + + final ResultData d = new ResultData(); + d.setIteration(0); + Map<String, String> resultsDictionary = null; + resultsDictionary = runTest(d, getSingleTestTimeoutValue()); + if (runningReducedGlitchesTest) { + // Special treatment, we want to upload values, but also indicate that pre-test + // conditions failed. We will set the glitches count and zero out the rest. + String[] testValuesToChangeArray = new String[] { + KEY_RESULT_RECORDER_BENCHMARK, + KEY_RESULT_RECORDER_OUTLIER, + KEY_RESULT_PLAYER_BENCHMARK, + KEY_RESULT_PLAYER_OUTLIER, + KEY_RESULT_RECORDER_BUFFER_CALLBACK, + KEY_RESULT_PLAYER_BUFFER_CALLBACK + }; + + for (String key : testValuesToChangeArray) { + final String metricsKey = getMetricsKey(key); + if (resultsDictionary.containsKey(metricsKey)) { + resultsDictionary.put(metricsKey, "0"); + } + } + } + + loopbackTestHelper.addTestData(d, resultsDictionary, false); + } + private void initializeTest(ITestInvocationListener listener) throws UnsupportedOperationException, DeviceNotAvailableException { @@ -370,7 +506,7 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { } private Map<String, String> runTest(ResultData data, final long timeout) - throws DeviceNotAvailableException { + throws DeviceNotAvailableException, TestFailureException { // start measurement and wait for result file final NullOutputReceiver receiver = new NullOutputReceiver(); @@ -434,7 +570,7 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { // Trust but verify, so get Audio Level from ADB and compare to value from app final int adbAudioLevel = - AudioLevelUtility.extractDeviceAudioLevelFromAdbShell(getDevice()); + AudioLevelUtility.extractDeviceHeadsetLevelFromAdbShell(getDevice()); if (data.getAudioLevel() != adbAudioLevel) { final String errMsg = String.format( @@ -461,7 +597,7 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { } private Map<String, String> uploadLogsReturnMetrics(ITestInvocationListener listener) - throws DeviceNotAvailableException { + throws DeviceNotAvailableException, TestFailureException { // "resultDictionary" is used to post results to dashboards like BlackBox // "results" contains test logs to be uploaded; i.e. to Sponge @@ -545,7 +681,7 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { return TestType.NONE; } - private boolean verifyTestParameters() { + private boolean verifyTestParameters() throws TestFailureException { if (getTestType() != TestType.NONE) { return true; } diff --git a/src/com/android/media/tests/AudioLoopbackTestHelper.java b/src/com/android/media/tests/AudioLoopbackTestHelper.java index 4e9c2b0..c9e058a 100644 --- a/src/com/android/media/tests/AudioLoopbackTestHelper.java +++ b/src/com/android/media/tests/AudioLoopbackTestHelper.java @@ -287,15 +287,20 @@ public class AudioLoopbackTestHelper { mAllResults = new ArrayList<ResultData>(iterations); } - public void addTestData(ResultData data, Map<String, String> resultDictionary) { + public void addTestData(ResultData data, + Map<String, + String> resultDictionary, + boolean useImageAnalyzer) { mResultDictionaries.add(data.getIteration(), resultDictionary); mAllResults.add(data); - // Analyze captured screenshot to see if wave form is within reason - final String screenshot = data.getLogFile(LogFileType.GRAPH); - final Pair<Result, String> result = AudioLoopbackImageAnalyzer.analyzeImage(screenshot); - data.setImageAnalyzerResult(result.first); - data.setFailureReason(result.second); + if (useImageAnalyzer) { + // Analyze captured screenshot to see if wave form is within reason + final String screenshot = data.getLogFile(LogFileType.GRAPH); + final Pair<Result, String> result = AudioLoopbackImageAnalyzer.analyzeImage(screenshot); + data.setImageAnalyzerResult(result.first); + data.setFailureReason(result.second); + } } public final List<ResultData> getAllTestData() { diff --git a/src/com/android/media/tests/TestFailureException.java b/src/com/android/media/tests/TestFailureException.java new file mode 100644 index 0000000..c33c195 --- /dev/null +++ b/src/com/android/media/tests/TestFailureException.java @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +/** Exception used to indicate test failure. */ +public class TestFailureException extends RuntimeException { + static final long serialVersionUID = 1L; + + public TestFailureException() { + super(); + } +} diff --git a/src/com/android/media/tests/TestRunHelper.java b/src/com/android/media/tests/TestRunHelper.java index f752cf3..f619772 100644 --- a/src/com/android/media/tests/TestRunHelper.java +++ b/src/com/android/media/tests/TestRunHelper.java @@ -39,18 +39,19 @@ public class TestRunHelper { return mTestStopTime - mTestStartTime; } - public void reportFailure(String errMsg) { + public void reportFailure(String errMsg) throws TestFailureException { CLog.e(errMsg); + mListener.testRunFailed(errMsg); mListener.testFailed(mTestId, errMsg); mListener.testEnded(mTestId, new HashMap<String, String>()); - mListener.testRunFailed(errMsg); + throw new TestFailureException(); } /** @param resultDictionary */ public void endTest(Map<String, String> resultDictionary) { mTestStopTime = System.currentTimeMillis(); - mListener.testEnded(mTestId, resultDictionary); mListener.testRunEnded(getTotalTestTime(), resultDictionary); + mListener.testEnded(mTestId, resultDictionary); } public void startTest(int numberOfTests) { |