diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/com/android/media/tests/AudioLevelUtility.java | 65 | ||||
| -rw-r--r-- | src/com/android/media/tests/AudioLoopbackImageAnalyzer.java | 475 | ||||
| -rw-r--r-- | src/com/android/media/tests/AudioLoopbackTest.java | 770 | ||||
| -rw-r--r-- | src/com/android/media/tests/AudioLoopbackTestHelper.java | 587 | ||||
| -rw-r--r-- | src/com/android/media/tests/TestRunHelper.java | 61 |
5 files changed, 1776 insertions, 182 deletions
diff --git a/src/com/android/media/tests/AudioLevelUtility.java b/src/com/android/media/tests/AudioLevelUtility.java new file mode 100644 index 0000000..3aee1fb --- /dev/null +++ b/src/com/android/media/tests/AudioLevelUtility.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +import com.android.ddmlib.CollectingOutputReceiver; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; + +import java.util.concurrent.TimeUnit; + +/** Class to provide audio level utility functions for a test device */ +public class AudioLevelUtility { + + public static int extractDeviceAudioLevelFromAdbShell(ITestDevice device) + throws DeviceNotAvailableException { + + final String ADB_SHELL_DUMPSYS_AUDIO = "dumpsys audio"; + final String STREAM_MUSIC = "- STREAM_MUSIC:"; + final String HEADSET = "(headset): "; + + final CollectingOutputReceiver receiver = new CollectingOutputReceiver(); + + device.executeShellCommand( + ADB_SHELL_DUMPSYS_AUDIO, receiver, 300, TimeUnit.MILLISECONDS, 1); + final String shellOutput = receiver.getOutput(); + if (shellOutput == null || shellOutput.isEmpty()) { + return -1; + } + + int audioLevel = -1; + int pos = shellOutput.indexOf(STREAM_MUSIC); + if (pos != -1) { + pos = shellOutput.indexOf(HEADSET, pos); + if (pos != -1) { + final int start = pos + HEADSET.length(); + final int stop = shellOutput.indexOf(",", start); + if (stop != -1) { + final String audioLevelStr = shellOutput.substring(start, stop); + try { + audioLevel = Integer.parseInt(audioLevelStr); + } catch (final NumberFormatException e) { + CLog.e(e.getMessage()); + audioLevel = 1; + } + } + } + } + + return audioLevel; + } +} diff --git a/src/com/android/media/tests/AudioLoopbackImageAnalyzer.java b/src/com/android/media/tests/AudioLoopbackImageAnalyzer.java new file mode 100644 index 0000000..3b6d767 --- /dev/null +++ b/src/com/android/media/tests/AudioLoopbackImageAnalyzer.java @@ -0,0 +1,475 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.util.Pair; + +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; + +import javax.imageio.ImageIO; + +/** + * Class that analyzes a screenshot captured from AudioLoopback test. There is a wave form in the + * screenshot that has specific colors (TARGET_COLOR). This class extracts those colors and analyzes + * wave amplitude, duration and form and make a decision if it's a legitimate wave form or not. + */ +public class AudioLoopbackImageAnalyzer { + + // General + private static final int HORIZONTAL_THRESHOLD = 10; + private static final int VERTICAL_THRESHOLD = 0; + private static final int PRIMARY_WAVE_COLOR = 0xFF1E4A99; + private static final int SECONDARY_WAVE_COLOR = 0xFF1D4998; + private static final int[] TARGET_COLORS_TABLET = + new int[] {PRIMARY_WAVE_COLOR, SECONDARY_WAVE_COLOR}; + private static final int[] TARGET_COLORS_PHONE = new int[] {PRIMARY_WAVE_COLOR}; + + private static final float EXPERIMENTAL_WAVE_MAX_TABLET = 69.0f; // In percent of image height + private static final float EXPERIMENTAL_WAVE_MAX_PHONE = 32.0f; // In percent of image height + + // Image + private static final int TABLET_SCREEN_MIN_WIDTH = 1700; + private static final int TABLET_SCREEN_MIN_HEIGHT = 2300; + + // Duration parameters + // Max duration should not span more than 2 of the 11 sections in the graph + // Min duration should not be less than 1/4 of a section + private static final float SECTION_WIDTH_IN_PERCENT = 100 * 1 / 11; // In percent of image width + private static final float DURATION_MIN = SECTION_WIDTH_IN_PERCENT / 4; + + // Amplitude + // Required numbers of column for a response + private static final int MIN_NUMBER_OF_COLUMNS = 4; + // The difference between two amplitude columns should not be more than this + private static final float MAX_ALLOWED_COLUMN_DECREASE = 0.50f; + // Only check MAX_ALLOWED_COLUMN_DECREASE up to this number + private static final float MIN_NUMBER_OF_DECREASING_COLUMNS = 8; + + enum Result { + PASS, + FAIL, + UNKNOWN + } + + private static class Amplitude { + public int maxHeight = -1; + public int zeroCounter = 0; + } + + public static Pair<Result, String> analyzeImage(String imgFile) { + final String FN_TAG = "AudioLoopbackImageAnalyzer.analyzeImage"; + + BufferedImage img = null; + try { + final File f = new File(imgFile); + img = ImageIO.read(f); + } catch (final IOException e) { + CLog.e(e); + throw new RuntimeException("Error loading image file '" + imgFile + "'"); + } + + final int width = img.getWidth(); + final int height = img.getHeight(); + + CLog.i("image width=" + width + ", height=" + height); + + // Compute thresholds and min/max values based on image witdh, height + final float waveMax; + final int[] targetColors; + final int amplitudeCenterMaxDiff; + final float maxDuration; + final int minNrOfZeroesBetweenAmplitudes; + + if (width >= TABLET_SCREEN_MIN_WIDTH && height >= TABLET_SCREEN_MIN_HEIGHT) { + CLog.i("Apply TABLET config values"); + waveMax = EXPERIMENTAL_WAVE_MAX_TABLET; + amplitudeCenterMaxDiff = 40; + minNrOfZeroesBetweenAmplitudes = 8; + maxDuration = 3 * SECTION_WIDTH_IN_PERCENT; + targetColors = TARGET_COLORS_TABLET; + } else { + waveMax = EXPERIMENTAL_WAVE_MAX_PHONE; + amplitudeCenterMaxDiff = 20; + minNrOfZeroesBetweenAmplitudes = 5; + maxDuration = 2.5f * SECTION_WIDTH_IN_PERCENT; + targetColors = TARGET_COLORS_PHONE; + } + + // Amplitude + // Max height should be about 80% of wave max. + // Min height should be about 40% of wave max. + final float AMPLITUDE_MAX_VALUE = waveMax * 0.8f; + final float AMPLITUDE_MIN_VALUE = waveMax * 0.4f; + + final int[] vertical = new int[height]; + final int[] horizontal = new int[width]; + + projectPixelsToXAxis(img, targetColors, horizontal, width, height); + filter(horizontal, HORIZONTAL_THRESHOLD); + final Pair<Integer, Integer> durationBounds = getBounds(horizontal); + if (!boundsWithinRange(durationBounds, 0, width)) { + final String fmt = "%1$s Upper/Lower bound along horizontal axis not found"; + final String err = String.format(fmt, FN_TAG); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + projectPixelsToYAxis(img, targetColors, vertical, height, durationBounds); + filter(vertical, VERTICAL_THRESHOLD); + final Pair<Integer, Integer> amplitudeBounds = getBounds(vertical); + if (!boundsWithinRange(durationBounds, 0, height)) { + final String fmt = "%1$s: Upper/Lower bound along vertical axis not found"; + final String err = String.format(fmt, FN_TAG); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + final int durationLeft = durationBounds.first.intValue(); + final int durationRight = durationBounds.second.intValue(); + final int amplitudeTop = amplitudeBounds.first.intValue(); + final int amplitudeBottom = amplitudeBounds.second.intValue(); + + final float amplitude = (amplitudeBottom - amplitudeTop) * 100.0f / height; + final float duration = (durationRight - durationLeft) * 100.0f / width; + + CLog.i("AudioLoopbackImageAnalyzer: Amplitude=" + amplitude + ", Duration=" + duration); + + Pair<Result, String> amplResult = + analyzeAmplitude( + vertical, + amplitude, + amplitudeTop, + amplitudeBottom, + AMPLITUDE_MIN_VALUE, + AMPLITUDE_MAX_VALUE, + amplitudeCenterMaxDiff); + if (amplResult.first != Result.PASS) { + return amplResult; + } + + amplResult = + analyzeDuration( + horizontal, + duration, + durationLeft, + durationRight, + DURATION_MIN, + maxDuration, + MIN_NUMBER_OF_COLUMNS, + minNrOfZeroesBetweenAmplitudes); + if (amplResult.first != Result.PASS) { + return amplResult; + } + + return new Pair<Result, String>(Result.PASS, ""); + } + + /** + * Function to analyze the waveforms duration (how wide it stretches along x-axis) and to make + * sure the waveform degrades nicely, i.e. the amplitude columns becomes smaller and smaller + * over time. + * + * @param horizontal - int array with waveforms amplitude values + * @param duration - calculated length of duration in percent of screen width + * @param durationLeft - index for "horizontal" where waveform starts + * @param durationRight - index for "horizontal" where waveform ends + * @param durationMin - if duration is below this value, return FAIL and failure reason + * @param durationMax - if duration exceed this value, return FAIL and failure reason + * @param minNumberOfAmplitudes - min number of amplitudes (columns) in waveform to pass test + * @param minNrOfZeroesBetweenAmplitudes - min number of required zeroes between amplitudes + * @return - returns result status and failure reason, if any + */ + private static Pair<Result, String> analyzeDuration( + int[] horizontal, + float duration, + int durationLeft, + int durationRight, + final float durationMin, + final float durationMax, + final int minNumberOfAmplitudes, + final int minNrOfZeroesBetweenAmplitudes) { + // This is the tricky one; basically, there should be "columns" that starts + // at "durationLeft", with the tallest column to the left and then column + // height will drop until it fades completely after "durationRight". + final String FN_TAG = "AudioLoopbackImageAnalyzer.analyzeDuration"; + + if (duration < durationMin || duration > durationMax) { + final String fmt = "%1$s: Duration outside range, value=%2$f, range=(%3$f,%4$f)"; + return handleError(fmt, FN_TAG, duration, durationMin, durationMax); + } + + final ArrayList<Amplitude> amplitudes = new ArrayList<Amplitude>(); + Amplitude currentAmplitude = null; + int zeroCounter = 0; + + for (int i = durationLeft; i < durationRight; i++) { + final int v = horizontal[i]; + if (v == 0) { + zeroCounter++; + } else { + CLog.i("index=" + i + ", v=" + v); + + if (zeroCounter > minNrOfZeroesBetweenAmplitudes) { + // Found a new amplitude; update old amplitude + // with the "gap" count - i.e. nr of zeroes between the amplitudes + if (currentAmplitude != null) { + currentAmplitude.zeroCounter = zeroCounter; + } + + // Create new Amplitude object + currentAmplitude = new Amplitude(); + amplitudes.add(currentAmplitude); + } + + // Reset counter + zeroCounter = 0; + + if (currentAmplitude != null && v > currentAmplitude.maxHeight) { + currentAmplitude.maxHeight = horizontal[i]; + } + } + } + + StringBuilder sb = new StringBuilder(128); + int counter = 0; + for (final Amplitude a : amplitudes) { + CLog.i( + sb.append("Amplitude=") + .append(counter) + .append(", MaxHeight=") + .append(a.maxHeight) + .append(", ZeroesToNextColumn=") + .append(a.zeroCounter) + .toString()); + counter++; + sb.setLength(0); + } + + if (amplitudes.size() < minNumberOfAmplitudes) { + final String fmt = "%1$s: Not enough amplitude columns, value=%2$d"; + return handleError(fmt, FN_TAG, amplitudes.size()); + } + + int currentColumnHeight = -1; + int oldColumnHeight = -1; + for (int i = 0; i < amplitudes.size(); i++) { + if (i == 0) { + oldColumnHeight = amplitudes.get(i).maxHeight; + continue; + } + + currentColumnHeight = amplitudes.get(i).maxHeight; + if (oldColumnHeight > currentColumnHeight) { + // We want at least a good number of columns that declines nicely. + // After MIN_NUMBER_OF_DECREASING_COLUMNS, we don't really care that much + if (i < MIN_NUMBER_OF_DECREASING_COLUMNS + && currentColumnHeight < (oldColumnHeight * MAX_ALLOWED_COLUMN_DECREASE)) { + final String fmt = + "%1$s: Amplitude column heights declined too much, " + + "old=%2$d, new=%3$d, column=%4$d"; + return handleError(fmt, FN_TAG, oldColumnHeight, currentColumnHeight, i); + } + oldColumnHeight = currentColumnHeight; + } else if (oldColumnHeight == currentColumnHeight) { + if (i < MIN_NUMBER_OF_DECREASING_COLUMNS) { + final String fmt = + "%1$s: Amplitude column heights are same, " + + "old=%2$d, new=%3$d, column=%4$d"; + return handleError(fmt, FN_TAG, oldColumnHeight, currentColumnHeight, i); + } + } else { + final String fmt = + "%1$s: Amplitude column heights don't decline, " + + "old=%2$d, new=%3$d, column=%4$d"; + return handleError(fmt, FN_TAG, oldColumnHeight, currentColumnHeight, i); + } + } + + return new Pair<Result, String>(Result.PASS, ""); + } + + /** + * Function to analyze the waveforms duration (how wide it stretches along x-axis) and to make + * sure the waveform degrades nicely, i.e. the amplitude columns becomes smaller and smaller + * over time. + * + * @param vertical - integer array with waveforms amplitude accumulated values + * @param amplitude - calculated height of amplitude in percent of screen height + * @param amplitudeTop - index in "vertical" array where waveform starts + * @param amplitudeBottom - index in "vertical" array where waveform ends + * @param amplitudeMin - if amplitude is below this value, return FAIL and failure reason + * @param amplitudeMax - if amplitude exceed this value, return FAIL and failure reason + * @param amplitudeCenterDiffThreshold - threshold to check that waveform is centered + * @return - returns result status and failure reason, if any + */ + private static Pair<Result, String> analyzeAmplitude( + int[] vertical, + float amplitude, + int amplitudeTop, + int amplitudeBottom, + final float amplitudeMin, + final float amplitudeMax, + final int amplitudeCenterDiffThreshold) { + final String FN_TAG = "AudioLoopbackImageAnalyzer.analyzeAmplitude"; + + if (amplitude < amplitudeMin || amplitude > amplitudeMax) { + final String fmt = "%1$s: Amplitude outside range, value=%2$f, range=(%3$f,%4$f)"; + final String err = String.format(fmt, FN_TAG, amplitude, amplitudeMin, amplitudeMax); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + // Are the amplitude top/bottom centered around the centerline? + final int amplitudeCenter = getAmplitudeCenter(vertical, amplitudeTop, amplitudeBottom); + final int topDiff = amplitudeCenter - amplitudeTop; + final int bottomDiff = amplitudeBottom - amplitudeCenter; + final int diff = Math.abs(topDiff - bottomDiff); + + if (diff < amplitudeCenterDiffThreshold) { + return new Pair<Result, String>(Result.PASS, ""); + } + + final String fmt = + "%1$s: Amplitude not centered topDiff=%2$d, bottomDiff=%3$d, " + + "center=%4$d, diff=%5$d"; + final String err = String.format(fmt, FN_TAG, topDiff, bottomDiff, amplitudeCenter, diff); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + private static int getAmplitudeCenter(int[] vertical, int amplitudeTop, int amplitudeBottom) { + int max = -1; + int center = -1; + for (int i = amplitudeTop; i < amplitudeBottom; i++) { + if (vertical[i] > max) { + max = vertical[i]; + center = i; + } + } + + return center; + } + + private static void projectPixelsToXAxis( + BufferedImage img, + final int[] targetColors, + int[] horizontal, + final int width, + final int height) { + // "Flatten image" by projecting target colors horizontally, + // counting number of found pixels in each column + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + final int color = img.getRGB(x, y); + for (final int targetColor : targetColors) { + if (color == targetColor) { + horizontal[x]++; + break; + } + } + } + } + } + + private static void projectPixelsToYAxis( + BufferedImage img, + final int[] targetColors, + int[] vertical, + int height, + Pair<Integer, Integer> horizontalMinMax) { + + final int min = horizontalMinMax.first.intValue(); + final int max = horizontalMinMax.second.intValue(); + + // "Flatten image" by projecting target colors (between min/max) vertically, + // counting number of found pixels in each row + + // Pass over y-axis, restricted to horizontalMin, horizontalMax + for (int y = 0; y < height; y++) { + for (int x = min; x <= max; x++) { + final int color = img.getRGB(x, y); + for (final int targetColor : targetColors) { + if (color == targetColor) { + vertical[y]++; + break; + } + } + } + } + } + + private static Pair<Integer, Integer> getBounds(int[] array) { + // Determine min, max + int min = -1; + for (int i = 0; i < array.length; i++) { + if (array[i] > 0) { + min = i; + break; + } + } + + int max = -1; + for (int i = array.length - 1; i >= 0; i--) { + if (array[i] > 0) { + max = i; + break; + } + } + + return new Pair<Integer, Integer>(Integer.valueOf(min), Integer.valueOf(max)); + } + + private static void filter(int[] array, final int threshold) { + // Filter horizontal array; set all values < threshold to 0 + for (int i = 0; i < array.length; i++) { + final int v = array[i]; + if (v != 0 && v <= threshold) { + array[i] = 0; + } + } + } + + private static boolean boundsWithinRange(Pair<Integer, Integer> bounds, int low, int high) { + return low <= bounds.first.intValue() + && bounds.first.intValue() < high + && low <= bounds.second.intValue() + && bounds.second.intValue() < high; + } + + private static Pair<Result, String> handleError(String fmt, String tag, int arg1) { + final String err = String.format(fmt, tag, arg1); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + private static Pair<Result, String> handleError( + String fmt, String tag, int arg1, int arg2, int arg3) { + final String err = String.format(fmt, tag, arg1, arg2, arg3); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + private static Pair<Result, String> handleError( + String fmt, String tag, float arg1, float arg2, float arg3) { + final String err = String.format(fmt, tag, arg1, arg2, arg3); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } +} diff --git a/src/com/android/media/tests/AudioLoopbackTest.java b/src/com/android/media/tests/AudioLoopbackTest.java index ce737fd..a9c787c 100644 --- a/src/com/android/media/tests/AudioLoopbackTest.java +++ b/src/com/android/media/tests/AudioLoopbackTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015 The Android Open Source Project + * Copyright (C) 2017 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,47 +13,57 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.android.media.tests; -import com.android.ddmlib.CollectingOutputReceiver; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +import com.android.ddmlib.NullOutputReceiver; import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.media.tests.AudioLoopbackTestHelper.LogFileType; +import com.android.media.tests.AudioLoopbackTestHelper.ResultData; import com.android.tradefed.config.Option; import com.android.tradefed.device.DeviceNotAvailableException; import com.android.tradefed.device.ITestDevice; import com.android.tradefed.log.LogUtil.CLog; import com.android.tradefed.result.FileInputStreamSource; import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; import com.android.tradefed.result.LogDataType; import com.android.tradefed.testtype.IDeviceTest; import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.FileUtil; import com.android.tradefed.util.RunUtil; -import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; import java.io.IOException; +import java.nio.file.Files; import java.util.Collections; +import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; /** - * A harness that launches Audio Loopback tool and reports result. + * Runs Audio Latency and Audio Glitch test and reports result. + * + * <p>Strategy for Audio Latency Stress test: RUN test 1000 times. In each iteration, collect result + * files from device, parse and collect data in a ResultData object that also keeps track of + * location to test files for a particular iteration. + * + * <p>ANALYZE test results to produce statistics for 1. Latency and Confidence (Min, Max, Mean, + * Median) 2. Create CSV file with test run data 3. Print bad test data to host log file 4. Get + * number of test runs with valid data to send to dashboard 5. Produce histogram in host log file; + * count number of test results that fall into 1 ms wide buckets. + * + * <p>UPLOAD test results + log files from “bad” runs; i.e. runs that is missing some or all result + * data. */ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { - private static final long TIMEOUT_MS = 5 * 60 * 1000; // 5 min - private static final long DEVICE_SYNC_MS = 5 * 60 * 1000; // 5 min - private static final long POLLING_INTERVAL_MS = 5 * 1000; - private static final int MAX_ATTEMPTS = 3; - private static final String TESTTYPE_LATENCY = "222"; - private static final String TESTTYPE_BUFFER = "223"; - - private static final Map<String, String> METRICS_KEY_MAP = createMetricsKeyMap(); - - private ITestDevice mDevice; - + //=================================================================== + // TEST OPTIONS + //=================================================================== @Option(name = "run-key", description = "Run key for the test") private String mRunKey = "AudioLoopback"; @@ -66,12 +76,16 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { @Option(name = "audio-thread", description = "Audio Thread for Loopback app") private String mAudioThread = "1"; - @Option(name = "audio-level", description = "Audio Level for Loopback app. " + - "A device specific param which makes waveform in loopback test hit 60% to 80% range") - private String mAudioLevel = "12"; + @Option( + name = "audio-level", + description = + "Audio Level for Loopback app. A device specific" + + "param which makes waveform in loopback test hit 60% to 80% range" + ) + private String mAudioLevel = "-1"; @Option(name = "test-type", description = "Test type to be executed") - private String mTestType = TESTTYPE_LATENCY; + private String mTestType = TESTTYPE_LATENCY_STR; @Option(name = "buffer-test-duration", description = "Buffer test duration in seconds") private String mBufferTestDuration = "10"; @@ -79,212 +93,604 @@ public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { @Option(name = "key-prefix", description = "Key Prefix for reporting") private String mKeyPrefix = "48000_Mic3_"; + @Option(name = "iterations", description = "Number of test iterations") + private int mIterations = 1; + + @Option(name = "baseline_latency", description = "") + private float mBaselineLatency = 0f; + + //=================================================================== + // CLASS VARIABLES + //=================================================================== + private static final Map<String, String> METRICS_KEY_MAP = createMetricsKeyMap(); + private Map<LogFileType, LogFileData> mFileDataKeyMap; + private ITestDevice mDevice; + private TestRunHelper mTestRunHelper; + private AudioLoopbackTestHelper mLoopbackTestHelper; + + //=================================================================== + // CONSTANTS + //=================================================================== + private static final String TESTTYPE_LATENCY_STR = "222"; + private static final String TESTTYPE_GLITCH_STR = "223"; + private static final long TIMEOUT_MS = 5 * 60 * 1000; // 5 min + private static final long DEVICE_SYNC_MS = 5 * 60 * 1000; // 5 min + private static final long POLLING_INTERVAL_MS = 5 * 1000; + private static final int MAX_ATTEMPTS = 3; + private static final int MAX_NR_OF_LOG_UPLOADS = 100; + + private static final int LATENCY_ITERATIONS_LOWER_BOUND = 1; + private static final int LATENCY_ITERATIONS_UPPER_BOUND = 10000; + private static final int GLITCH_ITERATIONS_LOWER_BOUND = 1; + private static final int GLITCH_ITERATIONS_UPPER_BOUND = 1; + private static final String DEVICE_TEMP_DIR_PATH = "/sdcard/"; - private static final String OUTPUT_FILENAME = "output_" + System.currentTimeMillis(); - private static final String OUTPUT_RESULT_TXT_PATH = - DEVICE_TEMP_DIR_PATH + OUTPUT_FILENAME + ".txt"; - private static final String OUTPUT_PNG_PATH = DEVICE_TEMP_DIR_PATH + OUTPUT_FILENAME + ".png"; - private static final String OUTPUT_WAV_PATH = DEVICE_TEMP_DIR_PATH + OUTPUT_FILENAME + ".wav"; - private static final String OUTPUT_PLAYER_BUFFER_PATH = - DEVICE_TEMP_DIR_PATH + OUTPUT_FILENAME + "_playerBufferPeriod.txt"; - private static final String OUTPUT_PLAYER_BUFFER_PNG_PATH = - DEVICE_TEMP_DIR_PATH + OUTPUT_FILENAME + "_playerBufferPeriod.png"; - private static final String OUTPUT_RECORDER_BUFFER_PATH = - DEVICE_TEMP_DIR_PATH + OUTPUT_FILENAME + "_recorderBufferPeriod.txt"; - private static final String OUTPUT_RECORDER_BUFFER_PNG_PATH = - DEVICE_TEMP_DIR_PATH + OUTPUT_FILENAME + "_recorderBufferPeriod.png"; - private static final String OUTPUT_GLITCH_PATH = - DEVICE_TEMP_DIR_PATH + OUTPUT_FILENAME + "_glitchMillis.txt"; + private static final String FMT_OUTPUT_PREFIX = "output_%1$d_" + System.currentTimeMillis(); + private static final String FMT_DEVICE_FILENAME = FMT_OUTPUT_PREFIX + "%2$s"; + private static final String FMT_DEVICE_PATH = DEVICE_TEMP_DIR_PATH + FMT_DEVICE_FILENAME; + private static final String AM_CMD = "am start -n org.drrickorang.loopback/.LoopbackActivity" + " --ei SF %s --es FileName %s --ei MicSource %s --ei AudioThread %s" + " --ei AudioLevel %s --ei TestType %s --ei BufferTestDuration %s"; - private static Map<String, String> createMetricsKeyMap() { - Map<String, String> result = new HashMap<>(); - result.put("LatencyMs", "latency_ms"); - result.put("LatencyConfidence", "latency_confidence"); - result.put("Recorder Benchmark", "recorder_benchmark"); - result.put("Recorder Number of Outliers", "recorder_outliers"); - result.put("Player Benchmark", "player_benchmark"); - result.put("Player Number of Outliers", "player_outliers"); - result.put("Total Number of Glitches", "number_of_glitches"); - result.put("kth% Late Recorder Buffer Callbacks", "late_recorder_callbacks"); - result.put("kth% Late Player Buffer Callbacks", "late_player_callbacks"); - result.put("Glitches Per Hour", "glitches_per_hour"); - return Collections.unmodifiableMap(result); - } + private static final String ERR_PARAMETER_OUT_OF_BOUNDS = + "Test parameter '%1$s' is out of bounds. Lower limit = %2$d, upper limit = %3$d"; + + private static final String KEY_RESULT_LATENCY_MS = "latency_ms"; + private static final String KEY_RESULT_LATENCY_CONFIDENCE = "latency_confidence"; + private static final String KEY_RESULT_RECORDER_BENCHMARK = "recorder_benchmark"; + private static final String KEY_RESULT_RECORDER_OUTLIER = "recorder_outliers"; + private static final String KEY_RESULT_PLAYER_BENCHMARK = "player_benchmark"; + private static final String KEY_RESULT_PLAYER_OUTLIER = "player_outliers"; + private static final String KEY_RESULT_NUMBER_OF_GLITCHES = "number_of_glitches"; + private static final String KEY_RESULT_RECORDER_BUFFER_CALLBACK = "late_recorder_callbacks"; + private static final String KEY_RESULT_PLAYER_BUFFER_CALLBACK = "late_player_callbacks"; + private static final String KEY_RESULT_GLITCHES_PER_HOUR = "glitches_per_hour"; + private static final String KEY_RESULT_TEST_STATUS = "test_status"; + private static final String KEY_RESULT_AUDIO_LEVEL = "audio_level"; + private static final String KEY_RESULT_RMS = "rms"; + private static final String KEY_RESULT_RMS_AVERAGE = "rms_average"; + private static final String KEY_RESULT_SAMPLING_FREQUENCY_CONFIDENCE = "sf"; + private static final String KEY_RESULT_PERIOD_CONFIDENCE = "period_confidence"; + private static final String KEY_RESULT_SAMPLING_BLOCK_SIZE = "bs"; + + private static final LogFileType[] LATENCY_TEST_LOGS = { + LogFileType.RESULT, + LogFileType.GRAPH, + LogFileType.WAVE, + LogFileType.PLAYER_BUFFER, + LogFileType.RECORDER_BUFFER, + LogFileType.LOGCAT + }; + + private static final LogFileType[] GLITCH_TEST_LOGS = { + LogFileType.RESULT, + LogFileType.GRAPH, + LogFileType.WAVE, + LogFileType.PLAYER_BUFFER, + LogFileType.PLAYER_BUFFER_HISTOGRAM, + LogFileType.RECORDER_BUFFER, + LogFileType.RECORDER_BUFFER_HISTOGRAM, + LogFileType.GLITCHES_MILLIS, + LogFileType.LOGCAT + }; /** - * {@inheritDoc} + * The Audio Latency and Audio Glitch test deals with many various types of log files. To be + * able to generate log files in a generic manner, this map is provided to get access to log + * file properties like log name prefix, log name file extension and log type (leveraging + * tradefed class LogDataType, used when uploading log). */ + private final synchronized Map<LogFileType, LogFileData> getLogFileDataKeyMap() { + if (mFileDataKeyMap != null) { + return mFileDataKeyMap; + } + + final Map<LogFileType, LogFileData> result = new HashMap<LogFileType, LogFileData>(); + + // Populate dictionary with info about various types of logfiles + LogFileData l = new LogFileData(".txt", "result", LogDataType.TEXT); + result.put(LogFileType.RESULT, l); + + l = new LogFileData(".png", "graph", LogDataType.PNG); + result.put(LogFileType.GRAPH, l); + + l = new LogFileData(".wav", "wave", LogDataType.UNKNOWN); + result.put(LogFileType.WAVE, l); + + l = new LogFileData("_playerBufferPeriod.txt", "player_buffer", LogDataType.TEXT); + result.put(LogFileType.PLAYER_BUFFER, l); + + l = new LogFileData("_playerBufferPeriod.png", "player_buffer_histogram", LogDataType.PNG); + result.put(LogFileType.PLAYER_BUFFER_HISTOGRAM, l); + + l = new LogFileData("_recorderBufferPeriod.txt", "recorder_buffer", LogDataType.TEXT); + result.put(LogFileType.RECORDER_BUFFER, l); + + l = + new LogFileData( + "_recorderBufferPeriod.png", "recorder_buffer_histogram", LogDataType.PNG); + result.put(LogFileType.RECORDER_BUFFER_HISTOGRAM, l); + + l = new LogFileData("_glitchMillis.txt", "wave", LogDataType.TEXT); + result.put(LogFileType.GLITCHES_MILLIS, l); + + l = new LogFileData(".txt", "logcat", LogDataType.TEXT); + result.put(LogFileType.LOGCAT, l); + + mFileDataKeyMap = Collections.unmodifiableMap(result); + return mFileDataKeyMap; + } + + private static final Map<String, String> createMetricsKeyMap() { + final Map<String, String> result = new HashMap<String, String>(); + + result.put("LatencyMs", KEY_RESULT_LATENCY_MS); + result.put("LatencyConfidence", KEY_RESULT_LATENCY_CONFIDENCE); + result.put("SF", KEY_RESULT_SAMPLING_FREQUENCY_CONFIDENCE); + result.put("Recorder Benchmark", KEY_RESULT_RECORDER_BENCHMARK); + result.put("Recorder Number of Outliers", KEY_RESULT_RECORDER_OUTLIER); + result.put("Player Benchmark", KEY_RESULT_PLAYER_BENCHMARK); + result.put("Player Number of Outliers", KEY_RESULT_PLAYER_OUTLIER); + result.put("Total Number of Glitches", KEY_RESULT_NUMBER_OF_GLITCHES); + result.put("kth% Late Recorder Buffer Callbacks", KEY_RESULT_RECORDER_BUFFER_CALLBACK); + result.put("kth% Late Player Buffer Callbacks", KEY_RESULT_PLAYER_BUFFER_CALLBACK); + result.put("Glitches Per Hour", KEY_RESULT_GLITCHES_PER_HOUR); + result.put("Test Status", KEY_RESULT_TEST_STATUS); + result.put("AudioLevel", KEY_RESULT_AUDIO_LEVEL); + result.put("RMS", KEY_RESULT_RMS); + result.put("Average", KEY_RESULT_RMS_AVERAGE); + result.put("PeriodConfidence", KEY_RESULT_PERIOD_CONFIDENCE); + result.put("BS", KEY_RESULT_SAMPLING_BLOCK_SIZE); + + return Collections.unmodifiableMap(result); + } + + //=================================================================== + // ENUMS + //=================================================================== + public enum TestType { + GLITCH, + LATENCY, + LATENCY_STRESS, + NONE + } + + //=================================================================== + // INNER CLASSES + //=================================================================== + public final class LogFileData { + private String fileExtension; + private String filePrefix; + private LogDataType logDataType; + + private LogFileData(String fileExtension, String filePrefix, LogDataType logDataType) { + this.fileExtension = fileExtension; + this.filePrefix = filePrefix; + this.logDataType = logDataType; + } + } + + //=================================================================== + // FUNCTIONS + //=================================================================== + + /** {@inheritDoc} */ @Override public void setDevice(ITestDevice device) { mDevice = device; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public ITestDevice getDevice() { return mDevice; } /** - * {@inheritDoc} + * Test Entry Point + * + * <p>{@inheritDoc} */ @Override public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + + initializeTest(listener); + + mTestRunHelper.startTest(1); + + try { + if (!verifyTestParameters()) { + return; + } + + // Stop logcat logging so we can record one logcat log per iteration + getDevice().stopLogcat(); + + // Run test iterations + for (int i = 0; i < mIterations; i++) { + CLog.i("---- Iteration " + i + " of " + (mIterations - 1) + " -----"); + + final ResultData d = new ResultData(); + d.setIteration(i); + Map<String, String> resultsDictionary = null; + resultsDictionary = runTest(d, getSingleTestTimeoutValue()); + + mLoopbackTestHelper.addTestData(d, resultsDictionary); + } + + mLoopbackTestHelper.processTestData(); + } finally { + mTestRunHelper.endTest(uploadLogsReturnMetrics(listener)); + deleteAllTempFiles(); + getDevice().startLogcat(); + } + } + + private void initializeTest(ITestInvocationListener listener) + throws UnsupportedOperationException, DeviceNotAvailableException { + + mFileDataKeyMap = getLogFileDataKeyMap(); TestIdentifier testId = new TestIdentifier(getClass().getCanonicalName(), mRunKey); - ITestDevice device = getDevice(); - // Wait device to settle - RunUtil.getDefault().sleep(DEVICE_SYNC_MS); - listener.testRunStarted(mRunKey, 0); - listener.testStarted(testId); + // Allocate helpers + mTestRunHelper = new TestRunHelper(listener, testId); + mLoopbackTestHelper = new AudioLoopbackTestHelper(mIterations); - long testStartTime = System.currentTimeMillis(); - Map<String, String> metrics = new HashMap<>(); + getDevice().disableKeyguard(); + getDevice().waitForDeviceAvailable(DEVICE_SYNC_MS); + + getDevice().setDate(new Date()); + CLog.i("syncing device time to host time"); + } + + private Map<String, String> runTest(ResultData data, final long timeout) + throws DeviceNotAvailableException { // start measurement and wait for result file - CollectingOutputReceiver receiver = new CollectingOutputReceiver(); - device.unlockDevice(); - String loopbackCmd = String.format( - AM_CMD, mSamplingFreq, OUTPUT_FILENAME, mMicSource, mAudioThread, - mAudioLevel, mTestType, mBufferTestDuration); - CLog.i("Running cmd: " + loopbackCmd); - device.executeShellCommand(loopbackCmd, receiver, - TIMEOUT_MS, TimeUnit.MILLISECONDS, MAX_ATTEMPTS); - long timeout = Long.parseLong(mBufferTestDuration) * 1000 + TIMEOUT_MS; - long loopbackStartTime = System.currentTimeMillis(); - boolean isTimedOut = false; - boolean isResultGenerated = false; + final NullOutputReceiver receiver = new NullOutputReceiver(); + + final String loopbackCmd = getTestCommand(data.getIteration()); + CLog.i("Loopback cmd: " + loopbackCmd); + + // Clear logcat + // Seems like getDevice().clearLogcat(); doesn't do anything? + // Do it through ADB + getDevice().executeAdbCommand("logcat", "-c"); + final long deviceTestStartTime = getDevice().getDeviceDate(); + + getDevice() + .executeShellCommand( + loopbackCmd, receiver, TIMEOUT_MS, TimeUnit.MILLISECONDS, MAX_ATTEMPTS); + + final long loopbackStartTime = System.currentTimeMillis(); File loopbackReport = null; - while (!isResultGenerated && !isTimedOut) { + + data.setDeviceTestStartTime(deviceTestStartTime); + + // Try to retrieve result file from device. + final String resultFilename = getDeviceFilename(LogFileType.RESULT, data.getIteration()); + do { RunUtil.getDefault().sleep(POLLING_INTERVAL_MS); - isTimedOut = (System.currentTimeMillis() - loopbackStartTime >= timeout); - boolean isResultFileFound = device.doesFileExist(OUTPUT_RESULT_TXT_PATH); - if (isResultFileFound) { - loopbackReport = device.pullFile(OUTPUT_RESULT_TXT_PATH); - if (loopbackReport.length() > 0) { - isResultGenerated = true; + if (getDevice().doesFileExist(resultFilename)) { + // Store device log files in tmp directory on Host and add to ResultData object + storeDeviceFilesOnHost(data); + final String reportFilename = data.getLogFile(LogFileType.RESULT); + if (reportFilename != null && !reportFilename.isEmpty()) { + loopbackReport = new File(reportFilename); + if (loopbackReport.length() > 0) { + break; + } } } - } - if (isTimedOut) { - reportFailure(listener, testId, "Loopback result not found, timed out."); - return; + data.setIsTimedOut(System.currentTimeMillis() - loopbackStartTime >= timeout); + } while (!data.hasLogFile(LogFileType.RESULT) && !data.isTimedOut()); + + // Grab logcat for iteration + final InputStreamSource lc = getDevice().getLogcatSince(deviceTestStartTime); + saveLogcatForIteration(data, lc, data.getIteration()); + + // Check if test timed out. If so, don't fail the test, but return to upper logic. + // We accept certain number of individual test timeouts. + if (data.isTimedOut()) { + // No device result files retrieved, so no need to parse + return null; } - // TODO: fail the test or rerun if the confidence level is too low + // parse result - CLog.i("== Loopback result =="); + Map<String, String> loopbackResult = null; + try { - Map<String, String> loopbackResult = parseResult(loopbackReport); - if (loopbackResult == null || loopbackResult.size() == 0) { - reportFailure(listener, testId, "Failed to parse Loopback result."); - return; + loopbackResult = + AudioLoopbackTestHelper.parseKeyValuePairFromFile( + loopbackReport, METRICS_KEY_MAP, "=", "%s: %s"); + populateResultData(loopbackResult, data); + + // Trust but verify, so get Audio Level from ADB and compare to value from app + final int adbAudioLevel = + AudioLevelUtility.extractDeviceAudioLevelFromAdbShell(getDevice()); + if (data.getAudioLevel() != adbAudioLevel) { + final String errMsg = + String.format( + "App Audio Level (%1$d)differs from ADB level (%2$d)", + data.getAudioLevel(), adbAudioLevel); + mTestRunHelper.reportFailure(errMsg); + } + } catch (final IOException ioe) { + CLog.e(ioe); + mTestRunHelper.reportFailure("I/O error while parsing Loopback result."); + } catch (final NumberFormatException ne) { + CLog.e(ne); + mTestRunHelper.reportFailure("Number format error parsing Loopback result."); + } + + return loopbackResult; + } + + private final long getSingleTestTimeoutValue() { + return Long.parseLong(mBufferTestDuration) * 1000 + TIMEOUT_MS; + } + + private Map<String, String> uploadLogsReturnMetrics(ITestInvocationListener listener) + throws DeviceNotAvailableException { + + // "resultDictionary" is used to post results to dashboards like BlackBox + // "results" contains test logs to be uploaded; i.e. to Sponge + + List<ResultData> results = null; + Map<String, String> resultDictionary = new HashMap<String, String>(); + + switch (getTestType()) { + case GLITCH: + case LATENCY: + // use dictionary collected from single test run + resultDictionary = mLoopbackTestHelper.getResultDictionaryForIteration(0); + results = mLoopbackTestHelper.getAllTestData(); + break; + case LATENCY_STRESS: + try { + saveResultsAsCSVFile(listener); + } catch (final IOException e) { + CLog.e(e); + } + + final int nrOfValidResults = mLoopbackTestHelper.processTestData(); + if (nrOfValidResults == 0) { + mTestRunHelper.reportFailure("No good data was collected"); + } + + final String nrOfTestsWithResults = Integer.toString(nrOfValidResults); + resultDictionary.put("NrOfTestsWithResults", nrOfTestsWithResults); + results = mLoopbackTestHelper.getWorstResults(MAX_NR_OF_LOG_UPLOADS); + break; + default: + break; + } + + // Upload relevant logs + for (final ResultData d : results) { + final LogFileType[] logFileTypes = getLogFileTypesForCurrentTest(); + for (final LogFileType logType : logFileTypes) { + uploadLog(listener, logType, d); + } + } + + return resultDictionary; + } + + private TestType getTestType() { + if (mTestType.equals(TESTTYPE_GLITCH_STR)) { + if (GLITCH_ITERATIONS_LOWER_BOUND <= mIterations + && mIterations <= GLITCH_ITERATIONS_UPPER_BOUND) { + return TestType.GLITCH; } - metrics = loopbackResult; - listener.testLog( - mKeyPrefix + "result", - LogDataType.TEXT, - new FileInputStreamSource(loopbackReport)); - File loopbackGraphFile = device.pullFile(OUTPUT_PNG_PATH); - listener.testLog( - mKeyPrefix + "graph", - LogDataType.PNG, - new FileInputStreamSource(loopbackGraphFile)); - File loopbackWaveFile = device.pullFile(OUTPUT_WAV_PATH); - listener.testLog( - mKeyPrefix + "wave", - LogDataType.UNKNOWN, - new FileInputStreamSource(loopbackWaveFile)); - if (mTestType.equals(TESTTYPE_BUFFER)) { - File loopbackPlayerBuffer = device.pullFile(OUTPUT_PLAYER_BUFFER_PATH); - listener.testLog( - mKeyPrefix + "player_buffer", - LogDataType.TEXT, - new FileInputStreamSource(loopbackPlayerBuffer)); - File loopbackPlayerBufferPng = device.pullFile(OUTPUT_PLAYER_BUFFER_PNG_PATH); - listener.testLog( - mKeyPrefix + "player_buffer_histogram", - LogDataType.PNG, - new FileInputStreamSource(loopbackPlayerBufferPng)); - - File loopbackRecorderBuffer = device.pullFile(OUTPUT_RECORDER_BUFFER_PATH); - listener.testLog( - mKeyPrefix + "recorder_buffer", - LogDataType.TEXT, - new FileInputStreamSource(loopbackRecorderBuffer)); - File loopbackRecorderBufferPng = device.pullFile(OUTPUT_RECORDER_BUFFER_PNG_PATH); - listener.testLog( - mKeyPrefix + "recorder_buffer_histogram", - LogDataType.PNG, - new FileInputStreamSource(loopbackRecorderBufferPng)); - - File loopbackGlitch = device.pullFile(OUTPUT_GLITCH_PATH); - listener.testLog( - mKeyPrefix + "glitches_millis", - LogDataType.TEXT, - new FileInputStreamSource(loopbackGlitch)); + } + + if (mTestType.equals(TESTTYPE_LATENCY_STR)) { + if (mIterations == 1) { + return TestType.LATENCY; } - } catch (IOException ioe) { - CLog.e(ioe.getMessage()); - reportFailure(listener, testId, "I/O error while parsing Loopback result."); + + if (LATENCY_ITERATIONS_LOWER_BOUND <= mIterations + && mIterations <= LATENCY_ITERATIONS_UPPER_BOUND) { + return TestType.LATENCY_STRESS; + } + } + + return TestType.NONE; + } + + private boolean verifyTestParameters() { + if (getTestType() != TestType.NONE) { + return true; + } + + if (mTestType.equals(TESTTYPE_GLITCH_STR) + && (mIterations < GLITCH_ITERATIONS_LOWER_BOUND + || mIterations > GLITCH_ITERATIONS_UPPER_BOUND)) { + final String error = + String.format( + ERR_PARAMETER_OUT_OF_BOUNDS, + "iterations", + GLITCH_ITERATIONS_LOWER_BOUND, + GLITCH_ITERATIONS_UPPER_BOUND); + mTestRunHelper.reportFailure(error); + return false; + } + + if (mTestType.equals(TESTTYPE_LATENCY_STR) + && (mIterations < LATENCY_ITERATIONS_LOWER_BOUND + || mIterations > LATENCY_ITERATIONS_UPPER_BOUND)) { + final String error = + String.format( + ERR_PARAMETER_OUT_OF_BOUNDS, + "iterations", + LATENCY_ITERATIONS_LOWER_BOUND, + LATENCY_ITERATIONS_UPPER_BOUND); + mTestRunHelper.reportFailure(error); + return false; + } + + return true; + } + + private void populateResultData(final Map<String, String> results, ResultData data) { + if (results == null || results.isEmpty()) { return; } - long durationMs = System.currentTimeMillis() - testStartTime; - listener.testEnded(testId, metrics); - listener.testRunEnded(durationMs, metrics); + String key = KEY_RESULT_LATENCY_MS; + if (results.containsKey(key)) { + data.setLatency(Float.parseFloat(results.get(key))); + } + + key = KEY_RESULT_LATENCY_CONFIDENCE; + if (results.containsKey(key)) { + data.setConfidence(Float.parseFloat(results.get(key))); + } + + key = KEY_RESULT_AUDIO_LEVEL; + if (results.containsKey(key)) { + data.setAudioLevel(Integer.parseInt(results.get(key))); + } + + key = KEY_RESULT_RMS; + if (results.containsKey(key)) { + data.setRMS(Float.parseFloat(results.get(key))); + } + + key = KEY_RESULT_RMS_AVERAGE; + if (results.containsKey(key)) { + data.setRMSAverage(Float.parseFloat(results.get(key))); + } + + key = KEY_RESULT_PERIOD_CONFIDENCE; + if (results.containsKey(key)) { + data.setPeriodConfidence(Float.parseFloat(results.get(key))); + } + + key = KEY_RESULT_SAMPLING_BLOCK_SIZE; + if (results.containsKey(key)) { + data.setBlockSize(Integer.parseInt(results.get(key))); + } } - /** - * Report failure with error message specified and fail the test. - * - * @param listener - * @param testId - * @param errMsg - */ - private void reportFailure(ITestInvocationListener listener, TestIdentifier testId, - String errMsg) { - CLog.e(errMsg); - listener.testFailed(testId, errMsg); - listener.testEnded(testId, new HashMap<String, String>()); - listener.testRunFailed(errMsg); + private void storeDeviceFilesOnHost(ResultData data) throws DeviceNotAvailableException { + final int iteration = data.getIteration(); + for (final LogFileType log : getLogFileTypesForCurrentTest()) { + if (getDevice().doesFileExist(getDeviceFilename(log, iteration))) { + final String deviceFileName = getDeviceFilename(log, iteration); + final File logFile = getDevice().pullFile(deviceFileName); + data.setLogFile(log, logFile.getAbsolutePath()); + CLog.i("Delete file from device: " + deviceFileName); + deleteFileFromDevice(deviceFileName); + } + } } - /** - * Parse result. - * Format: key = value - * - * @param result Loopback app result file - * @return a {@link HashMap} that contains metrics keys and results - * @throws IOException - */ - private Map<String, String> parseResult(File result) throws IOException { - Map<String, String> resultMap = new HashMap<>(); - BufferedReader br = new BufferedReader(new FileReader(result)); - try { - String line = br.readLine(); - while (line != null) { - line = line.trim().replaceAll(" +", " "); - String[] tokens = line.split("="); - if (tokens.length >= 2) { - String metricName = tokens[0].trim(); - String metricValue = tokens[1].trim(); - if (METRICS_KEY_MAP.containsKey(metricName)) { - CLog.i(String.format("%s: %s", metricName, metricValue)); - resultMap.put(mKeyPrefix + METRICS_KEY_MAP.get(metricName), metricValue); - } - } - line = br.readLine(); + private void deleteAllTempFiles() { + for (final ResultData d : mLoopbackTestHelper.getAllTestData()) { + final LogFileType[] logFileTypes = getLogFileTypesForCurrentTest(); + for (final LogFileType logType : logFileTypes) { + FileUtil.deleteFile(new File(d.getLogFile(logType))); } - } finally { - br.close(); } - return resultMap; } -}
\ No newline at end of file + + private void deleteFileFromDevice(String deviceFileName) throws DeviceNotAvailableException { + getDevice().executeShellCommand("rm -f " + deviceFileName); + } + + private final LogFileType[] getLogFileTypesForCurrentTest() { + switch (getTestType()) { + case GLITCH: + return GLITCH_TEST_LOGS; + case LATENCY: + case LATENCY_STRESS: + return LATENCY_TEST_LOGS; + default: + return null; + } + } + + private String getKeyPrefixForIteration(int iteration) { + if (mIterations == 1) { + // If only one run, skip the iteration number + return mKeyPrefix; + } + return mKeyPrefix + iteration + "_"; + } + + private String getDeviceFilename(LogFileType key, int iteration) { + final Map<LogFileType, LogFileData> map = getLogFileDataKeyMap(); + if (map.containsKey(key)) { + final LogFileData data = map.get(key); + return String.format(FMT_DEVICE_PATH, iteration, data.fileExtension); + } + return null; + } + + private void uploadLog(ITestInvocationListener listener, LogFileType key, ResultData data) { + final Map<LogFileType, LogFileData> map = getLogFileDataKeyMap(); + if (!map.containsKey(key)) { + return; + } + + final LogFileData logInfo = map.get(key); + final String prefix = getKeyPrefixForIteration(data.getIteration()) + logInfo.filePrefix; + final LogDataType logDataType = logInfo.logDataType; + File logFile = new File(data.getLogFile(key)); + InputStreamSource iss = new FileInputStreamSource(logFile); + listener.testLog(prefix, logDataType, iss); + + // cleanup + iss.cancel(); + } + + private void saveLogcatForIteration(ResultData data, InputStreamSource logcat, int iteration) { + if (logcat == null) { + CLog.i("Logcat could not be saved for iteration " + iteration); + return; + } + + //create a temp file + File temp; + try { + temp = FileUtil.createTempFile("logcat_" + iteration + "_", ".txt"); + data.setLogFile(LogFileType.LOGCAT, temp.getAbsolutePath()); + + // Copy logcat data into temp file + Files.copy(logcat.createInputStream(), temp.toPath(), REPLACE_EXISTING); + logcat.cancel(); + } catch (final IOException e) { + CLog.i("Error when saving logcat for iteration=" + iteration); + CLog.e(e); + } + } + + private void saveResultsAsCSVFile(ITestInvocationListener listener) + throws DeviceNotAvailableException, IOException { + final File csvTmpFile = File.createTempFile("audio_test_data", "csv"); + mLoopbackTestHelper.writeAllResultsToCSVFile(csvTmpFile, getDevice()); + InputStreamSource iss = new FileInputStreamSource(csvTmpFile); + listener.testLog("audio_test_data", LogDataType.JACOCO_CSV, iss); + + // cleanup + iss.cancel(); + csvTmpFile.delete(); + } + + private String getTestCommand(int currentIteration) { + return String.format( + AM_CMD, + mSamplingFreq, + String.format(FMT_OUTPUT_PREFIX, currentIteration), + mMicSource, + mAudioThread, + mAudioLevel, + mTestType, + mBufferTestDuration); + } +} diff --git a/src/com/android/media/tests/AudioLoopbackTestHelper.java b/src/com/android/media/tests/AudioLoopbackTestHelper.java new file mode 100644 index 0000000..a487d44 --- /dev/null +++ b/src/com/android/media/tests/AudioLoopbackTestHelper.java @@ -0,0 +1,587 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +import com.android.media.tests.AudioLoopbackImageAnalyzer.Result; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.util.Pair; + +import com.google.common.io.Files; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Helper class for AudioLoopbackTest. It keeps runtime data, analytics, */ +public class AudioLoopbackTestHelper { + + private StatisticsData mLatencyStats = null; + private StatisticsData mConfidenceStats = null; + private ArrayList<ResultData> mAllResults; + private ArrayList<ResultData> mGoodResults = new ArrayList<ResultData>(); + private ArrayList<ResultData> mBadResults = new ArrayList<ResultData>(); + private ArrayList<Map<String, String>> mResultDictionaries = + new ArrayList<Map<String, String>>(); + + // Controls acceptable tolerance in ms around median latency + private static final double TOLERANCE = 2.0; + + //=================================================================== + // ENUMS + //=================================================================== + public enum LogFileType { + RESULT, + WAVE, + GRAPH, + PLAYER_BUFFER, + PLAYER_BUFFER_HISTOGRAM, + RECORDER_BUFFER, + RECORDER_BUFFER_HISTOGRAM, + GLITCHES_MILLIS, + LOGCAT + } + + //=================================================================== + // INNER CLASSES + //=================================================================== + private class StatisticsData { + double mMin = 0; + double mMax = 0; + double mMean = 0; + double mMedian = 0; + + @Override + public String toString() { + return String.format( + "min = %1$f, max = %2$f, median=%3$f, mean = %4$f", mMin, mMax, mMedian, mMean); + } + } + + /** ResultData is an inner class that holds results and logfile info from each test run */ + public static class ResultData { + private Float mLatencyMs; + private Float mLatencyConfidence; + private Integer mAudioLevel; + private Integer mIteration; + private Long mDeviceTestStartTime; + private boolean mIsTimedOut = false; + private HashMap<LogFileType, String> mLogs = new HashMap<LogFileType, String>(); + private Result mImageAnalyzerResult = Result.UNKNOWN; + private String mFailureReason = null; + + // Optional + private Float mPeriodConfidence = Float.valueOf(0.0f); + private Float mRms = Float.valueOf(0.0f); + private Float mRmsAverage = Float.valueOf(0.0f); + private Integer mBblockSize = Integer.valueOf(0); + + public float getLatency() { + return mLatencyMs.floatValue(); + } + + public void setLatency(float latencyMs) { + this.mLatencyMs = Float.valueOf(latencyMs); + } + + public float getConfidence() { + return mLatencyConfidence.floatValue(); + } + + public void setConfidence(float latencyConfidence) { + this.mLatencyConfidence = Float.valueOf(latencyConfidence); + } + + public float getPeriodConfidence() { + return mPeriodConfidence.floatValue(); + } + + public void setPeriodConfidence(float periodConfidence) { + this.mPeriodConfidence = Float.valueOf(periodConfidence); + } + + public float getRMS() { + return mRms.floatValue(); + } + + public void setRMS(float rms) { + this.mRms = Float.valueOf(rms); + } + + public float getRMSAverage() { + return mRmsAverage.floatValue(); + } + + public void setRMSAverage(float rmsAverage) { + this.mRmsAverage = Float.valueOf(rmsAverage); + } + + public int getAudioLevel() { + return mAudioLevel.intValue(); + } + + public void setAudioLevel(int audioLevel) { + this.mAudioLevel = Integer.valueOf(audioLevel); + } + + public int getBlockSize() { + return mBblockSize.intValue(); + } + + public void setBlockSize(int blockSize) { + this.mBblockSize = Integer.valueOf(blockSize); + } + + public int getIteration() { + return mIteration.intValue(); + } + + public void setIteration(int iteration) { + this.mIteration = Integer.valueOf(iteration); + } + + public long getDeviceTestStartTime() { + return mDeviceTestStartTime.longValue(); + } + + public void setDeviceTestStartTime(long deviceTestStartTime) { + this.mDeviceTestStartTime = Long.valueOf(deviceTestStartTime); + } + + public Result getImageAnalyzerResult() { + return mImageAnalyzerResult; + } + + public void setImageAnalyzerResult(Result imageAnalyzerResult) { + this.mImageAnalyzerResult = imageAnalyzerResult; + } + + public String getFailureReason() { + return mFailureReason; + } + + public void setFailureReason(String failureReason) { + this.mFailureReason = failureReason; + } + + public boolean isTimedOut() { + return mIsTimedOut; + } + + public void setIsTimedOut(boolean isTimedOut) { + this.mIsTimedOut = isTimedOut; + } + + public String getLogFile(LogFileType log) { + return mLogs.get(log); + } + + public void setLogFile(LogFileType log, String filename) { + if (!mLogs.containsKey(log) && filename != null && !filename.isEmpty()) { + mLogs.put(log, filename); + } + } + + public boolean hasBadResults() { + return hasTimedOut() + || hasNoTestResults() + || hasNoLatencyResult() + || hasNoLatencyConfidence() + || mImageAnalyzerResult == Result.FAIL; + } + + public boolean hasTimedOut() { + return mIsTimedOut; + } + + public boolean hasLogFile(LogFileType log) { + return mLogs.containsKey(log); + } + + public boolean hasNoLatencyResult() { + return mLatencyMs == null; + } + + public boolean hasNoLatencyConfidence() { + return mLatencyConfidence == null; + } + + public boolean hasNoTestResults() { + return hasNoLatencyConfidence() && hasNoLatencyResult(); + } + + public static Comparator<ResultData> latencyComparator = + new Comparator<ResultData>() { + @Override + public int compare(ResultData o1, ResultData o2) { + return o1.mLatencyMs.compareTo(o2.mLatencyMs); + } + }; + + public static Comparator<ResultData> confidenceComparator = + new Comparator<ResultData>() { + @Override + public int compare(ResultData o1, ResultData o2) { + return o1.mLatencyConfidence.compareTo(o2.mLatencyConfidence); + } + }; + + public static Comparator<ResultData> iteratorComparator = + new Comparator<ResultData>() { + @Override + public int compare(ResultData o1, ResultData o2) { + return Integer.compare(o1.mIteration, o2.mIteration); + } + }; + + @Override + public String toString() { + final String NL = "\n"; + final StringBuilder sb = new StringBuilder(512); + sb.append("{").append(NL); + sb.append("{\nlatencyMs=").append(mLatencyMs).append(NL); + sb.append("latencyConfidence=").append(mLatencyConfidence).append(NL); + sb.append("isTimedOut=").append(mIsTimedOut).append(NL); + sb.append("iteration=").append(mIteration).append(NL); + sb.append("logs=").append(Arrays.toString(mLogs.values().toArray())).append(NL); + sb.append("audioLevel=").append(mAudioLevel).append(NL); + sb.append("deviceTestStartTime=").append(mDeviceTestStartTime).append(NL); + sb.append("rms=").append(mRms).append(NL); + sb.append("rmsAverage=").append(mRmsAverage).append(NL); + sb.append("}").append(NL); + return sb.toString(); + } + } + + public AudioLoopbackTestHelper(int iterations) { + mAllResults = new ArrayList<ResultData>(iterations); + } + + public void addTestData(ResultData data, Map<String, String> resultDictionary) { + mResultDictionaries.add(data.getIteration(), resultDictionary); + mAllResults.add(data); + + // Analyze captured screenshot to see if wave form is within reason + final String screenshot = data.getLogFile(LogFileType.GRAPH); + final Pair<Result, String> result = AudioLoopbackImageAnalyzer.analyzeImage(screenshot); + data.setImageAnalyzerResult(result.first); + data.setFailureReason(result.second); + } + + public final List<ResultData> getAllTestData() { + return mAllResults; + } + + public Map<String, String> getResultDictionaryForIteration(int i) { + return mResultDictionaries.get(i); + } + + /** + * Returns a list of the worst test result objects, up to maxNrOfWorstResults + * + * <p> + * + * <ol> + * <li> Tests in the bad results list are added first + * <li> If still space, add test results based on low confidence and then tests that are + * outside tolerance boundaries + * </ol> + * + * @param maxNrOfWorstResults + * @return list of worst test result objects + */ + public List<ResultData> getWorstResults(int maxNrOfWorstResults) { + int counter = 0; + final ArrayList<ResultData> worstResults = new ArrayList<ResultData>(maxNrOfWorstResults); + + for (final ResultData data : mBadResults) { + if (counter < maxNrOfWorstResults) { + worstResults.add(data); + counter++; + } + } + + for (final ResultData data : mGoodResults) { + if (counter < maxNrOfWorstResults) { + boolean failed = false; + if (data.getConfidence() < 1.0f) { + data.setFailureReason("Low confidence"); + failed = true; + } else if (data.getLatency() < (mLatencyStats.mMedian - TOLERANCE) + || data.getLatency() > (mLatencyStats.mMedian + TOLERANCE)) { + data.setFailureReason("Latency not within tolerance from median"); + failed = true; + } + + if (failed) { + worstResults.add(data); + counter++; + } + } + } + + return worstResults; + } + + public static Map<String, String> parseKeyValuePairFromFile( + File result, + final Map<String, String> dictionary, + final String splitOn, + final String keyValueFormat) + throws IOException { + + final Map<String, String> resultMap = new HashMap<String, String>(); + final BufferedReader br = Files.newReader(result, StandardCharsets.UTF_8); + + try { + String line = br.readLine(); + while (line != null) { + line = line.trim().replaceAll(" +", " "); + final String[] tokens = line.split(splitOn); + if (tokens.length >= 2) { + final String key = tokens[0].trim(); + final String value = tokens[1].trim(); + if (dictionary.containsKey(key)) { + CLog.i(String.format(keyValueFormat, key, value)); + resultMap.put(dictionary.get(key), value); + } + } + line = br.readLine(); + } + } finally { + br.close(); + } + return resultMap; + } + + public int processTestData() { + + // Collect statistics about the test run + int nrOfValidResults = 0; + double sumLatency = 0; + double sumConfidence = 0; + + final int totalNrOfTests = mAllResults.size(); + mLatencyStats = new StatisticsData(); + mConfidenceStats = new StatisticsData(); + mBadResults = new ArrayList<ResultData>(); + mGoodResults = new ArrayList<ResultData>(totalNrOfTests); + + // Copy all results into Good results list + mGoodResults.addAll(mAllResults); + + for (final ResultData data : mAllResults) { + if (data.hasBadResults()) { + mBadResults.add(data); + continue; + } + // Get mean values + sumLatency += data.getLatency(); + sumConfidence += data.getConfidence(); + } + + if (!mBadResults.isEmpty()) { + analyzeBadResults(mBadResults, mAllResults.size()); + } + + // Remove bad runs from result array + mGoodResults.removeAll(mBadResults); + + // Fail test immediately if we don't have ANY good results + if (mGoodResults.isEmpty()) { + return 0; + } + + nrOfValidResults = mGoodResults.size(); + + // ---- LATENCY: Get Median, Min and Max values ---- + Collections.sort(mGoodResults, ResultData.latencyComparator); + + mLatencyStats.mMin = mGoodResults.get(0).mLatencyMs; + mLatencyStats.mMax = mGoodResults.get(nrOfValidResults - 1).mLatencyMs; + mLatencyStats.mMean = sumLatency / nrOfValidResults; + // Is array even or odd numbered + if (nrOfValidResults % 2 == 0) { + final int middle = nrOfValidResults / 2; + final float middleLeft = mGoodResults.get(middle - 1).mLatencyMs; + final float middleRight = mGoodResults.get(middle).mLatencyMs; + mLatencyStats.mMedian = (middleLeft + middleRight) / 2.0f; + } else { + // It's and odd numbered array, just grab the middle value + mLatencyStats.mMedian = mGoodResults.get(nrOfValidResults / 2).mLatencyMs; + } + + // ---- CONFIDENCE: Get Median, Min and Max values ---- + Collections.sort(mGoodResults, ResultData.confidenceComparator); + + mConfidenceStats.mMin = mGoodResults.get(0).mLatencyConfidence; + mConfidenceStats.mMax = mGoodResults.get(nrOfValidResults - 1).mLatencyConfidence; + mConfidenceStats.mMean = sumConfidence / nrOfValidResults; + // Is array even or odd numbered + if (nrOfValidResults % 2 == 0) { + final int middle = nrOfValidResults / 2; + final float middleLeft = mGoodResults.get(middle - 1).mLatencyConfidence; + final float middleRight = mGoodResults.get(middle).mLatencyConfidence; + mConfidenceStats.mMedian = (middleLeft + middleRight) / 2.0f; + } else { + // It's and odd numbered array, just grab the middle value + mConfidenceStats.mMedian = mGoodResults.get(nrOfValidResults / 2).mLatencyConfidence; + } + + for (final ResultData data : mGoodResults) { + // Check if within Latency Tolerance + if (data.getConfidence() < 1.0f) { + data.setFailureReason("Low confidence"); + } else if (data.getLatency() < (mLatencyStats.mMedian - TOLERANCE) + || data.getLatency() > (mLatencyStats.mMedian + TOLERANCE)) { + data.setFailureReason("Latency not within tolerance from median"); + } + } + + // Create histogram + // Strategy: Create buckets based on whole ints, like 16 ms, 17 ms, 18 ms etc. Count how + // many tests fall into each bucket. Just cast the float to an int, no rounding up/down + // required. + final int[] histogram = new int[(int) mLatencyStats.mMax + 1]; + for (final ResultData rd : mGoodResults) { + // Increase value in bucket + histogram[(int) (rd.mLatencyMs.floatValue())]++; + } + + CLog.i("========== VALID RESULTS ============================================"); + CLog.i(String.format("Valid tests: %1$d of %2$d", nrOfValidResults, totalNrOfTests)); + CLog.i("Latency: " + mLatencyStats.toString()); + CLog.i("Confidence: " + mConfidenceStats.toString()); + CLog.i("========== HISTOGRAM ================================================"); + for (int i = 0; i < histogram.length; i++) { + if (histogram[i] > 0) { + CLog.i(String.format("%1$01d ms => %2$d", i, histogram[i])); + } + } + + // VERIFY the good results by running image analysis on the + // screenshot of the incoming audio waveform + + return nrOfValidResults; + } + + public void writeAllResultsToCSVFile(File csvFile, ITestDevice device) + throws DeviceNotAvailableException, FileNotFoundException, + UnsupportedEncodingException { + + final String deviceType = device.getProperty("ro.build.product"); + final String buildId = device.getBuildAlias(); + final String serialNumber = device.getSerialNumber(); + + // Sort data on iteration + Collections.sort(mAllResults, ResultData.iteratorComparator); + + final StringBuilder sb = new StringBuilder(256); + final PrintWriter writer = new PrintWriter(csvFile, StandardCharsets.UTF_8.name()); + final String SEPARATOR = ","; + + // Write column labels + writer.println( + "Device Time,Device Type,Build Id,Serial Number,Iteration,Latency," + + "Confidence,Period Confidence,Block Size,Audio Level,RMS,RMS Average," + + "Image Analysis,Failure Reason"); + for (final ResultData data : mAllResults) { + final Instant instant = Instant.ofEpochSecond(data.mDeviceTestStartTime); + + sb.append(instant).append(SEPARATOR); + sb.append(deviceType).append(SEPARATOR); + sb.append(buildId).append(SEPARATOR); + sb.append(serialNumber).append(SEPARATOR); + sb.append(data.getIteration()).append(SEPARATOR); + sb.append(data.getLatency()).append(SEPARATOR); + sb.append(data.getConfidence()).append(SEPARATOR); + sb.append(data.getPeriodConfidence()).append(SEPARATOR); + sb.append(data.getBlockSize()).append(SEPARATOR); + sb.append(data.getAudioLevel()).append(SEPARATOR); + sb.append(data.getRMS()).append(SEPARATOR); + sb.append(data.getRMSAverage()).append(SEPARATOR); + sb.append(data.getImageAnalyzerResult().name()).append(SEPARATOR); + sb.append(data.getFailureReason()); + + writer.println(sb.toString()); + + sb.setLength(0); + } + writer.close(); + } + + private void analyzeBadResults(ArrayList<ResultData> badResults, int totalNrOfTests) { + int testNoData = 0; + int testTimeoutCounts = 0; + int testResultsNotFoundCounts = 0; + int testWithoutLatencyResultCount = 0; + int testWithoutConfidenceResultCount = 0; + + for (final ResultData data : badResults) { + if (data.hasTimedOut()) { + testTimeoutCounts++; + testNoData++; + continue; + } + + if (data.hasNoTestResults()) { + testResultsNotFoundCounts++; + testNoData++; + continue; + } + + if (data.hasNoLatencyResult()) { + testWithoutLatencyResultCount++; + testNoData++; + continue; + } + + if (data.hasNoLatencyConfidence()) { + testWithoutConfidenceResultCount++; + testNoData++; + continue; + } + } + + CLog.i("========== BAD RESULTS ============================================"); + CLog.i(String.format("No Data: %1$d of %2$d", testNoData, totalNrOfTests)); + CLog.i(String.format("Timed out: %1$d of %2$d", testTimeoutCounts, totalNrOfTests)); + CLog.i( + String.format( + "No results: %1$d of %2$d", testResultsNotFoundCounts, totalNrOfTests)); + CLog.i( + String.format( + "No Latency results: %1$d of %2$d", + testWithoutLatencyResultCount, totalNrOfTests)); + CLog.i( + String.format( + "No Confidence results: %1$d of %2$d", + testWithoutConfidenceResultCount, totalNrOfTests)); + } +} diff --git a/src/com/android/media/tests/TestRunHelper.java b/src/com/android/media/tests/TestRunHelper.java new file mode 100644 index 0000000..f752cf3 --- /dev/null +++ b/src/com/android/media/tests/TestRunHelper.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; + +import java.util.HashMap; +import java.util.Map; + +/** Generic helper class for tests */ +public class TestRunHelper { + + private long mTestStartTime = -1; + private long mTestStopTime = -1; + private ITestInvocationListener mListener; + private TestIdentifier mTestId; + + public TestRunHelper(ITestInvocationListener listener, TestIdentifier testId) { + mListener = listener; + mTestId = testId; + } + + public long getTotalTestTime() { + return mTestStopTime - mTestStartTime; + } + + public void reportFailure(String errMsg) { + CLog.e(errMsg); + mListener.testFailed(mTestId, errMsg); + mListener.testEnded(mTestId, new HashMap<String, String>()); + mListener.testRunFailed(errMsg); + } + + /** @param resultDictionary */ + public void endTest(Map<String, String> resultDictionary) { + mTestStopTime = System.currentTimeMillis(); + mListener.testEnded(mTestId, resultDictionary); + mListener.testRunEnded(getTotalTestTime(), resultDictionary); + } + + public void startTest(int numberOfTests) { + mListener.testRunStarted(mTestId.getTestName(), numberOfTests); + mListener.testStarted(mTestId); + mTestStartTime = System.currentTimeMillis(); + } +} |