diff options
author | Kuan-Tung Pan <kuantung@google.com> | 2017-08-02 19:50:04 +0000 |
---|---|---|
committer | Kuan-Tung Pan <kuantung@google.com> | 2017-08-02 19:50:04 +0000 |
commit | fbd59a05057ad29f5689e2faa1a907370f5e9de0 (patch) | |
tree | 122fe96fb22d232092ec5226ab59d1e5288c7829 | |
parent | 14672dddc6362f1ae0857ee5255ef95628cb98db (diff) | |
download | contrib-fbd59a05057ad29f5689e2faa1a907370f5e9de0.tar.gz |
Revert "Revert "Merge prod-tests/src/com/android/media/tests/ from platform/tools/tradefederation to src/com/android/media/tests/ BUG:63819116""
This reverts commit 14672dddc6362f1ae0857ee5255ef95628cb98db.
Change-Id: Ie52f8d188281ff1351b40c309c6b19ae160ded4e
29 files changed, 8338 insertions, 0 deletions
diff --git a/src/com/android/media/tests/AdbScreenrecordTest.java b/src/com/android/media/tests/AdbScreenrecordTest.java new file mode 100644 index 0000000..eb50f50 --- /dev/null +++ b/src/com/android/media/tests/AdbScreenrecordTest.java @@ -0,0 +1,433 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +import com.android.ddmlib.CollectingOutputReceiver; +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.Option; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.IFileEntry; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.CommandResult; +import com.android.tradefed.util.CommandStatus; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.RunUtil; + +import java.io.File; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Tests adb command "screenrecord", i.e. "adb screenrecord [--size] [--bit-rate] [--time-limit]" + * + * <p>The test use the above command to record a video of DUT's screen. It then tries to verify that + * a video was actually recorded and that the video is a valid video file. It currently uses + * 'avprobe' to do the video analysis along with extracting parameters from the adb command's + * output. + */ +public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest { + + //=================================================================== + // TEST OPTIONS + //=================================================================== + @Option(name = "run-key", description = "Run key for the test") + private String mRunKey = "AdbScreenRecord"; + + @Option(name = "time-limit", description = "Recording time in seconds", isTimeVal = true) + private long mRecordTimeInSeconds = -1; + + @Option(name = "size", description = "Video Size: 'widthxheight', e.g. '1280x720'") + private String mVideoSize = null; + + @Option(name = "bit-rate", description = "Video bit rate in megabits per second, e.g. 4000000") + private long mBitRate = -1; + + //=================================================================== + // CLASS VARIABLES + //=================================================================== + private ITestDevice mDevice; + private TestRunHelper mTestRunHelper; + + //=================================================================== + // CONSTANTS + //=================================================================== + private static final long TEST_TIMEOUT_MS = 5 * 60 * 1000; // 5 min + private static final long DEVICE_SYNC_MS = 5 * 60 * 1000; // 5 min + private static final long POLLING_INTERVAL_MS = 5 * 1000; // 5 sec + private static final long CMD_TIMEOUT_MS = 5 * 1000; // 5 sec + private static final String ERR_OPTION_MALFORMED = "Test option %1$s is not correct [%2$s]"; + private static final String OPTION_TIME_LIMIT = "--time-limit"; + private static final String OPTION_SIZE = "--size"; + private static final String OPTION_BITRATE = "--bit-rate"; + private static final String RESULT_KEY_RECORDED_FRAMES = "recorded_frames"; + private static final String RESULT_KEY_RECORDED_LENGTH = "recorded_length"; + private static final String RESULT_KEY_VERIFIED_DURATION = "verified_duration"; + private static final String RESULT_KEY_VERIFIED_BITRATE = "verified_bitrate"; + private static final String TEST_FILE = "/sdcard/screenrecord_test.mp4"; + private static final String AVPROBE_NOT_INSTALLED = + "Program 'avprobe' is not installed on host '%1$s'"; + private static final String REGEX_IS_VIDEO_OK = + "Duration: (\\d\\d:\\d\\d:\\d\\d.\\d\\d).+bitrate: (\\d+ .b\\/s)"; + private static final String AVPROBE_STR = "avprobe"; + + //=================================================================== + // ENUMS + //=================================================================== + enum HOST_SOFTWARE { + AVPROBE + } + + @Override + public void setDevice(ITestDevice device) { + mDevice = device; + } + + @Override + public ITestDevice getDevice() { + return mDevice; + } + + /** Main test function invoked by test harness */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + initializeTest(listener); + + CLog.i("Verify required software is installed on host"); + verifyRequiredSoftwareIsInstalled(HOST_SOFTWARE.AVPROBE); + + mTestRunHelper.startTest(1); + + Map<String, String> resultsDictionary = new HashMap<String, String>(); + try { + CLog.i("Verify that test options are valid"); + if (!verifyTestParameters()) { + return; + } + + // "resultDictionary" can be used to post results to dashboards like BlackBox + resultsDictionary = runTest(resultsDictionary, TEST_TIMEOUT_MS); + } finally { + final String metricsStr = Arrays.toString(resultsDictionary.entrySet().toArray()); + CLog.i("Uploading metrics values:\n" + metricsStr); + mTestRunHelper.endTest(resultsDictionary); + } + } + + /** + * Test code that calls "adb screenrecord" and checks for pass/fail criterias + * + * <p> + * + * <ul> + * <li>1. Run adb screenrecord command + * <li>2. Wait until there is a video file; fail if none appears + * <li>3. Analyze adb output and extract recorded number of frames and video length + * <li>4. Pull recorded video file off device + * <li>5. Using avprobe, analyze video file and extract duration and bitrate + * <li>6. Return extracted results + * </ul> + * + * @throws DeviceNotAvailableException + */ + private Map<String, String> runTest(Map<String, String> results, final long timeout) + throws DeviceNotAvailableException { + final CollectingOutputReceiver receiver = new CollectingOutputReceiver(); + final String cmd = generateAdbScreenRecordCommand(); + final String deviceFileName = getAbsoluteFilename(); + + CLog.i("START Execute device shell command: '" + cmd + "'"); + getDevice().executeShellCommand(cmd, receiver, timeout, TimeUnit.MILLISECONDS, 3); + String adbOutput = receiver.getOutput(); + CLog.i(adbOutput); + CLog.i("END Execute device shell command"); + + CLog.i("Wait for recorded file: " + deviceFileName); + if (!waitForFile(getDevice(), timeout, deviceFileName)) { + mTestRunHelper.reportFailure("Recorded test file not found"); + // Since we don't have a file, no need to delete it; we can return here + return results; + } + + CLog.i("Get number of recorded frames and recorded length from adb output"); + if (!extractVideoDataFromAdbOutput(adbOutput, results)) { + deleteFileFromDevice(deviceFileName); + return results; + } + + CLog.i("Get duration and bitrate info from video file using '" + AVPROBE_STR + "'"); + try { + extractDurationAndBitrateFromVideoFileUsingAvprobe(deviceFileName, results); + } catch (ParseException e) { + throw new RuntimeException(e); + } + deleteFileFromDevice(deviceFileName); + return results; + } + + /** Convert a string on form HH:mm:ss.SS to nearest number of seconds */ + private long convertBitrateToKilobits(String bitrate) { + Matcher m = Pattern.compile("(\\d+) (.)b\\/s").matcher(bitrate); + if (!m.matches()) { + return -1; + } + + final String unit = m.group(2).toUpperCase(); + long factor = 1; + switch (unit) { + case "K": + factor = 1; + break; + case "M": + factor = 1000; + break; + case "G": + factor = 1000000; + break; + } + + long rate = Long.parseLong(m.group(1)); + + return rate * factor; + } + + /** + * Convert a string on form HH:mm:ss.SS to nearest number of seconds + * + * @throws ParseException + */ + private long convertDurationToMilliseconds(String duration) throws ParseException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SS"); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + Date convertedDate = sdf.parse("1970-01-01 " + duration); + return convertedDate.getTime(); + } + + /** + * Deletes a file off a device + * + * @param deviceFileName - path and filename to file to be deleted + * @throws DeviceNotAvailableException + */ + private void deleteFileFromDevice(String deviceFileName) throws DeviceNotAvailableException { + CLog.i("Delete file from device: " + deviceFileName); + getDevice().executeShellCommand("rm -f " + deviceFileName); + } + + /** + * Extracts duration and bitrate data from a video file + * + * @throws DeviceNotAvailableException + * @throws ParseException + */ + private boolean extractDurationAndBitrateFromVideoFileUsingAvprobe( + String deviceFileName, Map<String, String> results) + throws DeviceNotAvailableException, ParseException { + CLog.i("Check if the recorded file has some data in it: " + deviceFileName); + IFileEntry video = getDevice().getFileEntry(deviceFileName); + if (video == null || video.getFileEntry().getSizeValue() < 1) { + mTestRunHelper.reportFailure("Video Entry info failed"); + return false; + } + + final File recordedVideo = getDevice().pullFile(deviceFileName); + CLog.i("Recorded video file: " + recordedVideo.getAbsolutePath()); + + CommandResult result = + RunUtil.getDefault() + .runTimedCmd( + CMD_TIMEOUT_MS, + AVPROBE_STR, + "-loglevel", + "info", + recordedVideo.getAbsolutePath()); + + // Remove file from host machine + FileUtil.deleteFile(recordedVideo); + + if (result.getStatus() != CommandStatus.SUCCESS) { + mTestRunHelper.reportFailure(AVPROBE_STR + " command failed"); + return false; + } + + String data = result.getStderr(); + CLog.i("data: " + data); + if (data == null || data.isEmpty()) { + mTestRunHelper.reportFailure(AVPROBE_STR + " output data is empty"); + return false; + } + + Matcher m = Pattern.compile(REGEX_IS_VIDEO_OK).matcher(data); + if (!m.find()) { + final String errMsg = + "Video verification failed; no matching verification pattern found"; + mTestRunHelper.reportFailure(errMsg); + return false; + } + + String duration = m.group(1); + long durationInMilliseconds = convertDurationToMilliseconds(duration); + String bitrate = m.group(2); + long bitrateInKilobits = convertBitrateToKilobits(bitrate); + + results.put(RESULT_KEY_VERIFIED_DURATION, Long.toString(durationInMilliseconds / 1000)); + results.put(RESULT_KEY_VERIFIED_BITRATE, Long.toString(bitrateInKilobits)); + return true; + } + + /** Extracts recorded number of frames and recorded video length from adb output */ + private boolean extractVideoDataFromAdbOutput(String adbOutput, Map<String, String> results) { + final String regEx = "recorded (\\d+) frames in (\\d+) second"; + Matcher m = Pattern.compile(regEx).matcher(adbOutput); + if (!m.find()) { + mTestRunHelper.reportFailure("Regular Expression did not find recorded frames"); + return false; + } + + int recordedFrames = Integer.parseInt(m.group(1)); + int recordedLength = Integer.parseInt(m.group(2)); + CLog.i("Recorded frames: " + recordedFrames); + CLog.i("Recorded length: " + recordedLength); + if (recordedFrames <= 0) { + mTestRunHelper.reportFailure("No recorded frames detected"); + return false; + } + + results.put(RESULT_KEY_RECORDED_FRAMES, Integer.toString(recordedFrames)); + results.put(RESULT_KEY_RECORDED_LENGTH, Integer.toString(recordedLength)); + return true; + } + + /** Generates an adb command from passed in test options */ + private String generateAdbScreenRecordCommand() { + final String SPACE = " "; + StringBuilder sb = new StringBuilder(128); + sb.append("screenrecord --verbose ").append(getAbsoluteFilename()); + + // Add test options if they have been passed in to the test + if (mRecordTimeInSeconds != -1) { + final long timeLimit = TimeUnit.MILLISECONDS.toSeconds(mRecordTimeInSeconds); + sb.append(SPACE).append(OPTION_TIME_LIMIT).append(SPACE).append(timeLimit); + } + + if (mVideoSize != null) { + sb.append(SPACE).append(OPTION_SIZE).append(SPACE).append(mVideoSize); + } + + if (mBitRate != -1) { + sb.append(SPACE).append(OPTION_BITRATE).append(SPACE).append(mBitRate); + } + + return sb.toString(); + } + + /** Returns absolute path to device recorded video file */ + private String getAbsoluteFilename() { + return TEST_FILE; + } + + /** Performs test initialization steps */ + private void initializeTest(ITestInvocationListener listener) + throws UnsupportedOperationException, DeviceNotAvailableException { + TestIdentifier testId = new TestIdentifier(getClass().getCanonicalName(), mRunKey); + + // Allocate helpers + mTestRunHelper = new TestRunHelper(listener, testId); + + getDevice().disableKeyguard(); + getDevice().waitForDeviceAvailable(DEVICE_SYNC_MS); + + CLog.i("Sync device time to host time"); + getDevice().setDate(new Date()); + } + + /** Verifies that required software is installed on host machine */ + private void verifyRequiredSoftwareIsInstalled(HOST_SOFTWARE software) { + String swName = ""; + switch (software) { + case AVPROBE: + swName = AVPROBE_STR; + CommandResult result = + RunUtil.getDefault().runTimedCmd(CMD_TIMEOUT_MS, swName, "-version"); + String output = result.getStdout(); + if (result.getStatus() == CommandStatus.SUCCESS && output.startsWith(swName)) { + return; + } + break; + } + + CLog.i("Program '" + swName + "' not found, report test failure"); + String hostname = RunUtil.getDefault().runTimedCmd(CMD_TIMEOUT_MS, "hostname").getStdout(); + + String err = String.format(AVPROBE_NOT_INSTALLED, (hostname == null) ? "" : hostname); + throw new RuntimeException(err); + } + + /** Verifies that passed in test parameters are legitimate */ + private boolean verifyTestParameters() { + if (mRecordTimeInSeconds != -1 && mRecordTimeInSeconds < 1) { + final String error = + String.format(ERR_OPTION_MALFORMED, OPTION_TIME_LIMIT, mRecordTimeInSeconds); + mTestRunHelper.reportFailure(error); + return false; + } + + if (mVideoSize != null) { + final String videoSizeRegEx = "\\d+x\\d+"; + Matcher m = Pattern.compile(videoSizeRegEx).matcher(mVideoSize); + if (!m.matches()) { + final String error = String.format(ERR_OPTION_MALFORMED, OPTION_SIZE, mVideoSize); + mTestRunHelper.reportFailure(error); + return false; + } + } + + if (mBitRate != -1 && mBitRate < 1) { + final String error = String.format(ERR_OPTION_MALFORMED, OPTION_BITRATE, mBitRate); + mTestRunHelper.reportFailure(error); + return false; + } + + return true; + } + + /** Checks for existence of a file on the device */ + private static boolean waitForFile( + ITestDevice device, final long timeout, final String absoluteFilename) + throws DeviceNotAvailableException { + final long checkFileStartTime = System.currentTimeMillis(); + + do { + RunUtil.getDefault().sleep(POLLING_INTERVAL_MS); + if (device.doesFileExist(absoluteFilename)) { + return true; + } + } while (System.currentTimeMillis() - checkFileStartTime < timeout); + + return false; + } +} diff --git a/src/com/android/media/tests/AudioJitterTest.java b/src/com/android/media/tests/AudioJitterTest.java new file mode 100644 index 0000000..7058cc7 --- /dev/null +++ b/src/com/android/media/tests/AudioJitterTest.java @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.CollectingOutputReceiver; +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * A harness that launches AudioJitter tool and reports result. + */ +public class AudioJitterTest implements IDeviceTest, IRemoteTest { + + private static final String RUN_KEY = "audiojitter"; + private static final long TIMEOUT_MS = 5 * 60 * 1000; // 5 min + private static final int MAX_ATTEMPTS = 3; + private static final Map<String, String> METRICS_KEY_MAP = createMetricsKeyMap(); + + private ITestDevice mDevice; + + private static final String DEVICE_TEMPORARY_DIR_PATH = "/data/local/tmp/"; + private static final String JITTER_BINARY_FILENAME = "sljitter"; + private static final String JITTER_BINARY_DEVICE_PATH = + DEVICE_TEMPORARY_DIR_PATH + JITTER_BINARY_FILENAME; + + private static Map<String, String> createMetricsKeyMap() { + Map<String, String> result = new HashMap<String, String>(); + result.put("min_jitter_ticks", "min_jitter_ticks"); + result.put("min_jitter_ms", "min_jitter_ms"); + result.put("min_jitter_period_id", "min_jitter_period_id"); + result.put("max_jitter_ticks", "max_jitter_ticks"); + result.put("max_jitter_ms", "max_jitter_ms"); + result.put("max_jitter_period_id", "max_jitter_period_id"); + result.put("mark_jitter_ticks", "mark_jitter_ticks"); + result.put("mark_jitter_ms", "mark_jitter_ms"); + result.put("max_cb_done_delay_ms", "max_cb_done_delay_ms"); + result.put("max_thread_delay_ms", "max_thread_delay_ms"); + result.put("max_render_delay_ms", "max_render_delay_ms"); + result.put("drift_rate", "drift_rate"); + result.put("error_ms", "error_ms"); + result.put("min_error_ms", "min_error_ms"); + result.put("max_error_ms", "max_error_ms"); + return Collections.unmodifiableMap(result); + } + + /** + * {@inheritDoc} + */ + @Override + public void setDevice(ITestDevice device) { + mDevice = device; + } + + /** + * {@inheritDoc} + */ + @Override + public ITestDevice getDevice() { + return mDevice; + } + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + TestIdentifier testId = new TestIdentifier(getClass().getCanonicalName(), RUN_KEY); + ITestDevice device = getDevice(); + + listener.testRunStarted(RUN_KEY, 0); + listener.testStarted(testId); + + long testStartTime = System.currentTimeMillis(); + Map<String, String> metrics = new HashMap<String, String>(); + String errMsg = null; + + // start jitter and wait for process to complete + CollectingOutputReceiver receiver = new CollectingOutputReceiver(); + device.executeShellCommand(JITTER_BINARY_DEVICE_PATH, receiver, + TIMEOUT_MS, TimeUnit.MILLISECONDS, MAX_ATTEMPTS); + String resultStr = receiver.getOutput(); + + if (resultStr != null) { + // parse result + CLog.i("== Jitter result =="); + Map<String, String> jitterResult = parseResult(resultStr); + if (jitterResult == null) { + errMsg = "Failed to parse Jitter result."; + } else { + metrics = jitterResult; + } + } else { + errMsg = "Jitter result not found."; + } + + if (errMsg != null) { + CLog.e(errMsg); + listener.testFailed(testId, errMsg); + listener.testEnded(testId, metrics); + listener.testRunFailed(errMsg); + } else { + long durationMs = System.currentTimeMillis() - testStartTime; + listener.testEnded(testId, metrics); + listener.testRunEnded(durationMs, metrics); + } + } + + /** + * Parse Jitter result. + * + * @param result Jitter result output + * @return a {@link HashMap} that contains metrics keys and results + */ + private Map<String, String> parseResult(String result) { + Map<String, String> resultMap = new HashMap<String, String>(); + String lines[] = result.split("\\r?\\n"); + for (String line: lines) { + line = line.trim().replaceAll(" +", " "); + String[] tokens = line.split(" "); + if (tokens.length >= 2) { + String metricName = tokens[0]; + String metricValue = tokens[1]; + if (METRICS_KEY_MAP.containsKey(metricName)) { + CLog.i(String.format("%s: %s", metricName, metricValue)); + resultMap.put(METRICS_KEY_MAP.get(metricName), metricValue); + } + } + } + return resultMap; + } +} diff --git a/src/com/android/media/tests/AudioLevelUtility.java b/src/com/android/media/tests/AudioLevelUtility.java new file mode 100644 index 0000000..3aee1fb --- /dev/null +++ b/src/com/android/media/tests/AudioLevelUtility.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +import com.android.ddmlib.CollectingOutputReceiver; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; + +import java.util.concurrent.TimeUnit; + +/** Class to provide audio level utility functions for a test device */ +public class AudioLevelUtility { + + public static int extractDeviceAudioLevelFromAdbShell(ITestDevice device) + throws DeviceNotAvailableException { + + final String ADB_SHELL_DUMPSYS_AUDIO = "dumpsys audio"; + final String STREAM_MUSIC = "- STREAM_MUSIC:"; + final String HEADSET = "(headset): "; + + final CollectingOutputReceiver receiver = new CollectingOutputReceiver(); + + device.executeShellCommand( + ADB_SHELL_DUMPSYS_AUDIO, receiver, 300, TimeUnit.MILLISECONDS, 1); + final String shellOutput = receiver.getOutput(); + if (shellOutput == null || shellOutput.isEmpty()) { + return -1; + } + + int audioLevel = -1; + int pos = shellOutput.indexOf(STREAM_MUSIC); + if (pos != -1) { + pos = shellOutput.indexOf(HEADSET, pos); + if (pos != -1) { + final int start = pos + HEADSET.length(); + final int stop = shellOutput.indexOf(",", start); + if (stop != -1) { + final String audioLevelStr = shellOutput.substring(start, stop); + try { + audioLevel = Integer.parseInt(audioLevelStr); + } catch (final NumberFormatException e) { + CLog.e(e.getMessage()); + audioLevel = 1; + } + } + } + } + + return audioLevel; + } +} diff --git a/src/com/android/media/tests/AudioLoopbackImageAnalyzer.java b/src/com/android/media/tests/AudioLoopbackImageAnalyzer.java new file mode 100644 index 0000000..3b6d767 --- /dev/null +++ b/src/com/android/media/tests/AudioLoopbackImageAnalyzer.java @@ -0,0 +1,475 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.util.Pair; + +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; + +import javax.imageio.ImageIO; + +/** + * Class that analyzes a screenshot captured from AudioLoopback test. There is a wave form in the + * screenshot that has specific colors (TARGET_COLOR). This class extracts those colors and analyzes + * wave amplitude, duration and form and make a decision if it's a legitimate wave form or not. + */ +public class AudioLoopbackImageAnalyzer { + + // General + private static final int HORIZONTAL_THRESHOLD = 10; + private static final int VERTICAL_THRESHOLD = 0; + private static final int PRIMARY_WAVE_COLOR = 0xFF1E4A99; + private static final int SECONDARY_WAVE_COLOR = 0xFF1D4998; + private static final int[] TARGET_COLORS_TABLET = + new int[] {PRIMARY_WAVE_COLOR, SECONDARY_WAVE_COLOR}; + private static final int[] TARGET_COLORS_PHONE = new int[] {PRIMARY_WAVE_COLOR}; + + private static final float EXPERIMENTAL_WAVE_MAX_TABLET = 69.0f; // In percent of image height + private static final float EXPERIMENTAL_WAVE_MAX_PHONE = 32.0f; // In percent of image height + + // Image + private static final int TABLET_SCREEN_MIN_WIDTH = 1700; + private static final int TABLET_SCREEN_MIN_HEIGHT = 2300; + + // Duration parameters + // Max duration should not span more than 2 of the 11 sections in the graph + // Min duration should not be less than 1/4 of a section + private static final float SECTION_WIDTH_IN_PERCENT = 100 * 1 / 11; // In percent of image width + private static final float DURATION_MIN = SECTION_WIDTH_IN_PERCENT / 4; + + // Amplitude + // Required numbers of column for a response + private static final int MIN_NUMBER_OF_COLUMNS = 4; + // The difference between two amplitude columns should not be more than this + private static final float MAX_ALLOWED_COLUMN_DECREASE = 0.50f; + // Only check MAX_ALLOWED_COLUMN_DECREASE up to this number + private static final float MIN_NUMBER_OF_DECREASING_COLUMNS = 8; + + enum Result { + PASS, + FAIL, + UNKNOWN + } + + private static class Amplitude { + public int maxHeight = -1; + public int zeroCounter = 0; + } + + public static Pair<Result, String> analyzeImage(String imgFile) { + final String FN_TAG = "AudioLoopbackImageAnalyzer.analyzeImage"; + + BufferedImage img = null; + try { + final File f = new File(imgFile); + img = ImageIO.read(f); + } catch (final IOException e) { + CLog.e(e); + throw new RuntimeException("Error loading image file '" + imgFile + "'"); + } + + final int width = img.getWidth(); + final int height = img.getHeight(); + + CLog.i("image width=" + width + ", height=" + height); + + // Compute thresholds and min/max values based on image witdh, height + final float waveMax; + final int[] targetColors; + final int amplitudeCenterMaxDiff; + final float maxDuration; + final int minNrOfZeroesBetweenAmplitudes; + + if (width >= TABLET_SCREEN_MIN_WIDTH && height >= TABLET_SCREEN_MIN_HEIGHT) { + CLog.i("Apply TABLET config values"); + waveMax = EXPERIMENTAL_WAVE_MAX_TABLET; + amplitudeCenterMaxDiff = 40; + minNrOfZeroesBetweenAmplitudes = 8; + maxDuration = 3 * SECTION_WIDTH_IN_PERCENT; + targetColors = TARGET_COLORS_TABLET; + } else { + waveMax = EXPERIMENTAL_WAVE_MAX_PHONE; + amplitudeCenterMaxDiff = 20; + minNrOfZeroesBetweenAmplitudes = 5; + maxDuration = 2.5f * SECTION_WIDTH_IN_PERCENT; + targetColors = TARGET_COLORS_PHONE; + } + + // Amplitude + // Max height should be about 80% of wave max. + // Min height should be about 40% of wave max. + final float AMPLITUDE_MAX_VALUE = waveMax * 0.8f; + final float AMPLITUDE_MIN_VALUE = waveMax * 0.4f; + + final int[] vertical = new int[height]; + final int[] horizontal = new int[width]; + + projectPixelsToXAxis(img, targetColors, horizontal, width, height); + filter(horizontal, HORIZONTAL_THRESHOLD); + final Pair<Integer, Integer> durationBounds = getBounds(horizontal); + if (!boundsWithinRange(durationBounds, 0, width)) { + final String fmt = "%1$s Upper/Lower bound along horizontal axis not found"; + final String err = String.format(fmt, FN_TAG); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + projectPixelsToYAxis(img, targetColors, vertical, height, durationBounds); + filter(vertical, VERTICAL_THRESHOLD); + final Pair<Integer, Integer> amplitudeBounds = getBounds(vertical); + if (!boundsWithinRange(durationBounds, 0, height)) { + final String fmt = "%1$s: Upper/Lower bound along vertical axis not found"; + final String err = String.format(fmt, FN_TAG); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + final int durationLeft = durationBounds.first.intValue(); + final int durationRight = durationBounds.second.intValue(); + final int amplitudeTop = amplitudeBounds.first.intValue(); + final int amplitudeBottom = amplitudeBounds.second.intValue(); + + final float amplitude = (amplitudeBottom - amplitudeTop) * 100.0f / height; + final float duration = (durationRight - durationLeft) * 100.0f / width; + + CLog.i("AudioLoopbackImageAnalyzer: Amplitude=" + amplitude + ", Duration=" + duration); + + Pair<Result, String> amplResult = + analyzeAmplitude( + vertical, + amplitude, + amplitudeTop, + amplitudeBottom, + AMPLITUDE_MIN_VALUE, + AMPLITUDE_MAX_VALUE, + amplitudeCenterMaxDiff); + if (amplResult.first != Result.PASS) { + return amplResult; + } + + amplResult = + analyzeDuration( + horizontal, + duration, + durationLeft, + durationRight, + DURATION_MIN, + maxDuration, + MIN_NUMBER_OF_COLUMNS, + minNrOfZeroesBetweenAmplitudes); + if (amplResult.first != Result.PASS) { + return amplResult; + } + + return new Pair<Result, String>(Result.PASS, ""); + } + + /** + * Function to analyze the waveforms duration (how wide it stretches along x-axis) and to make + * sure the waveform degrades nicely, i.e. the amplitude columns becomes smaller and smaller + * over time. + * + * @param horizontal - int array with waveforms amplitude values + * @param duration - calculated length of duration in percent of screen width + * @param durationLeft - index for "horizontal" where waveform starts + * @param durationRight - index for "horizontal" where waveform ends + * @param durationMin - if duration is below this value, return FAIL and failure reason + * @param durationMax - if duration exceed this value, return FAIL and failure reason + * @param minNumberOfAmplitudes - min number of amplitudes (columns) in waveform to pass test + * @param minNrOfZeroesBetweenAmplitudes - min number of required zeroes between amplitudes + * @return - returns result status and failure reason, if any + */ + private static Pair<Result, String> analyzeDuration( + int[] horizontal, + float duration, + int durationLeft, + int durationRight, + final float durationMin, + final float durationMax, + final int minNumberOfAmplitudes, + final int minNrOfZeroesBetweenAmplitudes) { + // This is the tricky one; basically, there should be "columns" that starts + // at "durationLeft", with the tallest column to the left and then column + // height will drop until it fades completely after "durationRight". + final String FN_TAG = "AudioLoopbackImageAnalyzer.analyzeDuration"; + + if (duration < durationMin || duration > durationMax) { + final String fmt = "%1$s: Duration outside range, value=%2$f, range=(%3$f,%4$f)"; + return handleError(fmt, FN_TAG, duration, durationMin, durationMax); + } + + final ArrayList<Amplitude> amplitudes = new ArrayList<Amplitude>(); + Amplitude currentAmplitude = null; + int zeroCounter = 0; + + for (int i = durationLeft; i < durationRight; i++) { + final int v = horizontal[i]; + if (v == 0) { + zeroCounter++; + } else { + CLog.i("index=" + i + ", v=" + v); + + if (zeroCounter > minNrOfZeroesBetweenAmplitudes) { + // Found a new amplitude; update old amplitude + // with the "gap" count - i.e. nr of zeroes between the amplitudes + if (currentAmplitude != null) { + currentAmplitude.zeroCounter = zeroCounter; + } + + // Create new Amplitude object + currentAmplitude = new Amplitude(); + amplitudes.add(currentAmplitude); + } + + // Reset counter + zeroCounter = 0; + + if (currentAmplitude != null && v > currentAmplitude.maxHeight) { + currentAmplitude.maxHeight = horizontal[i]; + } + } + } + + StringBuilder sb = new StringBuilder(128); + int counter = 0; + for (final Amplitude a : amplitudes) { + CLog.i( + sb.append("Amplitude=") + .append(counter) + .append(", MaxHeight=") + .append(a.maxHeight) + .append(", ZeroesToNextColumn=") + .append(a.zeroCounter) + .toString()); + counter++; + sb.setLength(0); + } + + if (amplitudes.size() < minNumberOfAmplitudes) { + final String fmt = "%1$s: Not enough amplitude columns, value=%2$d"; + return handleError(fmt, FN_TAG, amplitudes.size()); + } + + int currentColumnHeight = -1; + int oldColumnHeight = -1; + for (int i = 0; i < amplitudes.size(); i++) { + if (i == 0) { + oldColumnHeight = amplitudes.get(i).maxHeight; + continue; + } + + currentColumnHeight = amplitudes.get(i).maxHeight; + if (oldColumnHeight > currentColumnHeight) { + // We want at least a good number of columns that declines nicely. + // After MIN_NUMBER_OF_DECREASING_COLUMNS, we don't really care that much + if (i < MIN_NUMBER_OF_DECREASING_COLUMNS + && currentColumnHeight < (oldColumnHeight * MAX_ALLOWED_COLUMN_DECREASE)) { + final String fmt = + "%1$s: Amplitude column heights declined too much, " + + "old=%2$d, new=%3$d, column=%4$d"; + return handleError(fmt, FN_TAG, oldColumnHeight, currentColumnHeight, i); + } + oldColumnHeight = currentColumnHeight; + } else if (oldColumnHeight == currentColumnHeight) { + if (i < MIN_NUMBER_OF_DECREASING_COLUMNS) { + final String fmt = + "%1$s: Amplitude column heights are same, " + + "old=%2$d, new=%3$d, column=%4$d"; + return handleError(fmt, FN_TAG, oldColumnHeight, currentColumnHeight, i); + } + } else { + final String fmt = + "%1$s: Amplitude column heights don't decline, " + + "old=%2$d, new=%3$d, column=%4$d"; + return handleError(fmt, FN_TAG, oldColumnHeight, currentColumnHeight, i); + } + } + + return new Pair<Result, String>(Result.PASS, ""); + } + + /** + * Function to analyze the waveforms duration (how wide it stretches along x-axis) and to make + * sure the waveform degrades nicely, i.e. the amplitude columns becomes smaller and smaller + * over time. + * + * @param vertical - integer array with waveforms amplitude accumulated values + * @param amplitude - calculated height of amplitude in percent of screen height + * @param amplitudeTop - index in "vertical" array where waveform starts + * @param amplitudeBottom - index in "vertical" array where waveform ends + * @param amplitudeMin - if amplitude is below this value, return FAIL and failure reason + * @param amplitudeMax - if amplitude exceed this value, return FAIL and failure reason + * @param amplitudeCenterDiffThreshold - threshold to check that waveform is centered + * @return - returns result status and failure reason, if any + */ + private static Pair<Result, String> analyzeAmplitude( + int[] vertical, + float amplitude, + int amplitudeTop, + int amplitudeBottom, + final float amplitudeMin, + final float amplitudeMax, + final int amplitudeCenterDiffThreshold) { + final String FN_TAG = "AudioLoopbackImageAnalyzer.analyzeAmplitude"; + + if (amplitude < amplitudeMin || amplitude > amplitudeMax) { + final String fmt = "%1$s: Amplitude outside range, value=%2$f, range=(%3$f,%4$f)"; + final String err = String.format(fmt, FN_TAG, amplitude, amplitudeMin, amplitudeMax); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + // Are the amplitude top/bottom centered around the centerline? + final int amplitudeCenter = getAmplitudeCenter(vertical, amplitudeTop, amplitudeBottom); + final int topDiff = amplitudeCenter - amplitudeTop; + final int bottomDiff = amplitudeBottom - amplitudeCenter; + final int diff = Math.abs(topDiff - bottomDiff); + + if (diff < amplitudeCenterDiffThreshold) { + return new Pair<Result, String>(Result.PASS, ""); + } + + final String fmt = + "%1$s: Amplitude not centered topDiff=%2$d, bottomDiff=%3$d, " + + "center=%4$d, diff=%5$d"; + final String err = String.format(fmt, FN_TAG, topDiff, bottomDiff, amplitudeCenter, diff); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + private static int getAmplitudeCenter(int[] vertical, int amplitudeTop, int amplitudeBottom) { + int max = -1; + int center = -1; + for (int i = amplitudeTop; i < amplitudeBottom; i++) { + if (vertical[i] > max) { + max = vertical[i]; + center = i; + } + } + + return center; + } + + private static void projectPixelsToXAxis( + BufferedImage img, + final int[] targetColors, + int[] horizontal, + final int width, + final int height) { + // "Flatten image" by projecting target colors horizontally, + // counting number of found pixels in each column + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + final int color = img.getRGB(x, y); + for (final int targetColor : targetColors) { + if (color == targetColor) { + horizontal[x]++; + break; + } + } + } + } + } + + private static void projectPixelsToYAxis( + BufferedImage img, + final int[] targetColors, + int[] vertical, + int height, + Pair<Integer, Integer> horizontalMinMax) { + + final int min = horizontalMinMax.first.intValue(); + final int max = horizontalMinMax.second.intValue(); + + // "Flatten image" by projecting target colors (between min/max) vertically, + // counting number of found pixels in each row + + // Pass over y-axis, restricted to horizontalMin, horizontalMax + for (int y = 0; y < height; y++) { + for (int x = min; x <= max; x++) { + final int color = img.getRGB(x, y); + for (final int targetColor : targetColors) { + if (color == targetColor) { + vertical[y]++; + break; + } + } + } + } + } + + private static Pair<Integer, Integer> getBounds(int[] array) { + // Determine min, max + int min = -1; + for (int i = 0; i < array.length; i++) { + if (array[i] > 0) { + min = i; + break; + } + } + + int max = -1; + for (int i = array.length - 1; i >= 0; i--) { + if (array[i] > 0) { + max = i; + break; + } + } + + return new Pair<Integer, Integer>(Integer.valueOf(min), Integer.valueOf(max)); + } + + private static void filter(int[] array, final int threshold) { + // Filter horizontal array; set all values < threshold to 0 + for (int i = 0; i < array.length; i++) { + final int v = array[i]; + if (v != 0 && v <= threshold) { + array[i] = 0; + } + } + } + + private static boolean boundsWithinRange(Pair<Integer, Integer> bounds, int low, int high) { + return low <= bounds.first.intValue() + && bounds.first.intValue() < high + && low <= bounds.second.intValue() + && bounds.second.intValue() < high; + } + + private static Pair<Result, String> handleError(String fmt, String tag, int arg1) { + final String err = String.format(fmt, tag, arg1); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + private static Pair<Result, String> handleError( + String fmt, String tag, int arg1, int arg2, int arg3) { + final String err = String.format(fmt, tag, arg1, arg2, arg3); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } + + private static Pair<Result, String> handleError( + String fmt, String tag, float arg1, float arg2, float arg3) { + final String err = String.format(fmt, tag, arg1, arg2, arg3); + CLog.w(err); + return new Pair<Result, String>(Result.FAIL, err); + } +} diff --git a/src/com/android/media/tests/AudioLoopbackTest.java b/src/com/android/media/tests/AudioLoopbackTest.java new file mode 100644 index 0000000..1d9bab5 --- /dev/null +++ b/src/com/android/media/tests/AudioLoopbackTest.java @@ -0,0 +1,749 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +import com.android.ddmlib.NullOutputReceiver; +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.media.tests.AudioLoopbackTestHelper.LogFileType; +import com.android.media.tests.AudioLoopbackTestHelper.ResultData; +import com.android.tradefed.config.Option; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.FileInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.RunUtil; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Runs Audio Latency and Audio Glitch test and reports result. + * + * <p>Strategy for Audio Latency Stress test: RUN test 1000 times. In each iteration, collect result + * files from device, parse and collect data in a ResultData object that also keeps track of + * location to test files for a particular iteration. + * + * <p>ANALYZE test results to produce statistics for 1. Latency and Confidence (Min, Max, Mean, + * Median) 2. Create CSV file with test run data 3. Print bad test data to host log file 4. Get + * number of test runs with valid data to send to dashboard 5. Produce histogram in host log file; + * count number of test results that fall into 1 ms wide buckets. + * + * <p>UPLOAD test results + log files from “bad” runs; i.e. runs that is missing some or all result + * data. + */ +public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { + + //=================================================================== + // TEST OPTIONS + //=================================================================== + @Option(name = "run-key", description = "Run key for the test") + private String mRunKey = "AudioLoopback"; + + @Option(name = "sampling-freq", description = "Sampling Frequency for Loopback app") + private String mSamplingFreq = "48000"; + + @Option(name = "mic-source", description = "Mic Source for Loopback app") + private String mMicSource = "3"; + + @Option(name = "audio-thread", description = "Audio Thread for Loopback app") + private String mAudioThread = "1"; + + @Option( + name = "audio-level", + description = + "Audio Level for Loopback app. A device specific" + + "param which makes waveform in loopback test hit 60% to 80% range" + ) + private String mAudioLevel = "-1"; + + @Option(name = "test-type", description = "Test type to be executed") + private String mTestType = TESTTYPE_LATENCY_STR; + + @Option(name = "buffer-test-duration", description = "Buffer test duration in seconds") + private String mBufferTestDuration = "10"; + + @Option(name = "key-prefix", description = "Key Prefix for reporting") + private String mKeyPrefix = "48000_Mic3_"; + + @Option(name = "iterations", description = "Number of test iterations") + private int mIterations = 1; + + @Option(name = "baseline_latency", description = "") + private float mBaselineLatency = 0f; + + //=================================================================== + // CLASS VARIABLES + //=================================================================== + private static final Map<String, String> METRICS_KEY_MAP = createMetricsKeyMap(); + private Map<LogFileType, LogFileData> mFileDataKeyMap; + private ITestDevice mDevice; + private TestRunHelper mTestRunHelper; + private AudioLoopbackTestHelper mLoopbackTestHelper; + + //=================================================================== + // CONSTANTS + //=================================================================== + private static final String TESTTYPE_LATENCY_STR = "222"; + private static final String TESTTYPE_GLITCH_STR = "223"; + private static final long TIMEOUT_MS = 5 * 60 * 1000; // 5 min + private static final long DEVICE_SYNC_MS = 5 * 60 * 1000; // 5 min + private static final long POLLING_INTERVAL_MS = 5 * 1000; + private static final int MAX_ATTEMPTS = 3; + private static final int MAX_NR_OF_LOG_UPLOADS = 100; + + private static final int LATENCY_ITERATIONS_LOWER_BOUND = 1; + private static final int LATENCY_ITERATIONS_UPPER_BOUND = 10000; + private static final int GLITCH_ITERATIONS_LOWER_BOUND = 1; + private static final int GLITCH_ITERATIONS_UPPER_BOUND = 1; + + private static final String DEVICE_TEMP_DIR_PATH = "/sdcard/"; + private static final String FMT_OUTPUT_PREFIX = "output_%1$d_" + System.currentTimeMillis(); + private static final String FMT_DEVICE_FILENAME = FMT_OUTPUT_PREFIX + "%2$s"; + private static final String FMT_DEVICE_PATH = DEVICE_TEMP_DIR_PATH + FMT_DEVICE_FILENAME; + + private static final String AM_CMD = + "am start -n org.drrickorang.loopback/.LoopbackActivity" + + " --ei SF %s --es FileName %s --ei MicSource %s --ei AudioThread %s" + + " --ei AudioLevel %s --ei TestType %s --ei BufferTestDuration %s"; + + private static final String ERR_PARAMETER_OUT_OF_BOUNDS = + "Test parameter '%1$s' is out of bounds. Lower limit = %2$d, upper limit = %3$d"; + + private static final String KEY_RESULT_LATENCY_MS = "latency_ms"; + private static final String KEY_RESULT_LATENCY_CONFIDENCE = "latency_confidence"; + private static final String KEY_RESULT_RECORDER_BENCHMARK = "recorder_benchmark"; + private static final String KEY_RESULT_RECORDER_OUTLIER = "recorder_outliers"; + private static final String KEY_RESULT_PLAYER_BENCHMARK = "player_benchmark"; + private static final String KEY_RESULT_PLAYER_OUTLIER = "player_outliers"; + private static final String KEY_RESULT_NUMBER_OF_GLITCHES = "number_of_glitches"; + private static final String KEY_RESULT_RECORDER_BUFFER_CALLBACK = "late_recorder_callbacks"; + private static final String KEY_RESULT_PLAYER_BUFFER_CALLBACK = "late_player_callbacks"; + private static final String KEY_RESULT_GLITCHES_PER_HOUR = "glitches_per_hour"; + private static final String KEY_RESULT_TEST_STATUS = "test_status"; + private static final String KEY_RESULT_AUDIO_LEVEL = "audio_level"; + private static final String KEY_RESULT_RMS = "rms"; + private static final String KEY_RESULT_RMS_AVERAGE = "rms_average"; + private static final String KEY_RESULT_SAMPLING_FREQUENCY_CONFIDENCE = "sampling_frequency"; + private static final String KEY_RESULT_PERIOD_CONFIDENCE = "period_confidence"; + private static final String KEY_RESULT_SAMPLING_BLOCK_SIZE = "block_size"; + + private static final LogFileType[] LATENCY_TEST_LOGS = { + LogFileType.RESULT, + LogFileType.GRAPH, + LogFileType.WAVE, + LogFileType.PLAYER_BUFFER, + LogFileType.PLAYER_BUFFER_HISTOGRAM, + LogFileType.PLAYER_BUFFER_PERIOD_TIMES, + LogFileType.RECORDER_BUFFER, + LogFileType.RECORDER_BUFFER_HISTOGRAM, + LogFileType.RECORDER_BUFFER_PERIOD_TIMES, + LogFileType.LOGCAT + }; + + private static final LogFileType[] GLITCH_TEST_LOGS = { + LogFileType.RESULT, + LogFileType.GRAPH, + LogFileType.WAVE, + LogFileType.PLAYER_BUFFER, + LogFileType.PLAYER_BUFFER_HISTOGRAM, + LogFileType.PLAYER_BUFFER_PERIOD_TIMES, + LogFileType.RECORDER_BUFFER, + LogFileType.RECORDER_BUFFER_HISTOGRAM, + LogFileType.RECORDER_BUFFER_PERIOD_TIMES, + LogFileType.GLITCHES_MILLIS, + LogFileType.HEAT_MAP, + LogFileType.LOGCAT + }; + + /** + * The Audio Latency and Audio Glitch test deals with many various types of log files. To be + * able to generate log files in a generic manner, this map is provided to get access to log + * file properties like log name prefix, log name file extension and log type (leveraging + * tradefed class LogDataType, used when uploading log). + */ + private final synchronized Map<LogFileType, LogFileData> getLogFileDataKeyMap() { + if (mFileDataKeyMap != null) { + return mFileDataKeyMap; + } + + final Map<LogFileType, LogFileData> result = new HashMap<LogFileType, LogFileData>(); + + // Populate dictionary with info about various types of logfiles + LogFileData l = new LogFileData(".txt", "result", LogDataType.TEXT); + result.put(LogFileType.RESULT, l); + + l = new LogFileData(".png", "graph", LogDataType.PNG); + result.put(LogFileType.GRAPH, l); + + l = new LogFileData(".wav", "wave", LogDataType.UNKNOWN); + result.put(LogFileType.WAVE, l); + + l = new LogFileData("_playerBufferPeriod.txt", "player_buffer", LogDataType.TEXT); + result.put(LogFileType.PLAYER_BUFFER, l); + + l = new LogFileData("_playerBufferPeriod.png", "player_buffer_histogram", LogDataType.PNG); + result.put(LogFileType.PLAYER_BUFFER_HISTOGRAM, l); + + String fileExtension = "_playerBufferPeriodTimes.txt"; + String uploadName = "player_buffer_period_times"; + l = new LogFileData(fileExtension, uploadName, LogDataType.TEXT); + result.put(LogFileType.PLAYER_BUFFER_PERIOD_TIMES, l); + + l = new LogFileData("_recorderBufferPeriod.txt", "recorder_buffer", LogDataType.TEXT); + result.put(LogFileType.RECORDER_BUFFER, l); + + fileExtension = "_recorderBufferPeriod.png"; + uploadName = "recorder_buffer_histogram"; + l = new LogFileData(fileExtension, uploadName, LogDataType.PNG); + result.put(LogFileType.RECORDER_BUFFER_HISTOGRAM, l); + + fileExtension = "_recorderBufferPeriodTimes.txt"; + uploadName = "recorder_buffer_period_times"; + l = new LogFileData(fileExtension, uploadName, LogDataType.TEXT); + result.put(LogFileType.RECORDER_BUFFER_PERIOD_TIMES, l); + + l = new LogFileData("_glitchMillis.txt", "glitches_millis", LogDataType.TEXT); + result.put(LogFileType.GLITCHES_MILLIS, l); + + + l = new LogFileData("_heatMap.png", "heat_map", LogDataType.PNG); + result.put(LogFileType.HEAT_MAP, l); + + l = new LogFileData(".txt", "logcat", LogDataType.TEXT); + result.put(LogFileType.LOGCAT, l); + + mFileDataKeyMap = Collections.unmodifiableMap(result); + return mFileDataKeyMap; + } + + private static final Map<String, String> createMetricsKeyMap() { + final Map<String, String> result = new HashMap<String, String>(); + + result.put("LatencyMs", KEY_RESULT_LATENCY_MS); + result.put("LatencyConfidence", KEY_RESULT_LATENCY_CONFIDENCE); + result.put("SF", KEY_RESULT_SAMPLING_FREQUENCY_CONFIDENCE); + result.put("Recorder Benchmark", KEY_RESULT_RECORDER_BENCHMARK); + result.put("Recorder Number of Outliers", KEY_RESULT_RECORDER_OUTLIER); + result.put("Player Benchmark", KEY_RESULT_PLAYER_BENCHMARK); + result.put("Player Number of Outliers", KEY_RESULT_PLAYER_OUTLIER); + result.put("Total Number of Glitches", KEY_RESULT_NUMBER_OF_GLITCHES); + result.put("kth% Late Recorder Buffer Callbacks", KEY_RESULT_RECORDER_BUFFER_CALLBACK); + result.put("kth% Late Player Buffer Callbacks", KEY_RESULT_PLAYER_BUFFER_CALLBACK); + result.put("Glitches Per Hour", KEY_RESULT_GLITCHES_PER_HOUR); + result.put("Test Status", KEY_RESULT_TEST_STATUS); + result.put("AudioLevel", KEY_RESULT_AUDIO_LEVEL); + result.put("RMS", KEY_RESULT_RMS); + result.put("Average", KEY_RESULT_RMS_AVERAGE); + result.put("PeriodConfidence", KEY_RESULT_PERIOD_CONFIDENCE); + result.put("BS", KEY_RESULT_SAMPLING_BLOCK_SIZE); + + return Collections.unmodifiableMap(result); + } + + //=================================================================== + // ENUMS + //=================================================================== + public enum TestType { + GLITCH, + LATENCY, + LATENCY_STRESS, + NONE + } + + //=================================================================== + // INNER CLASSES + //=================================================================== + public final class LogFileData { + private String fileExtension; + private String filePrefix; + private LogDataType logDataType; + + private LogFileData(String fileExtension, String filePrefix, LogDataType logDataType) { + this.fileExtension = fileExtension; + this.filePrefix = filePrefix; + this.logDataType = logDataType; + } + } + + //=================================================================== + // FUNCTIONS + //=================================================================== + + /** {@inheritDoc} */ + @Override + public void setDevice(ITestDevice device) { + mDevice = device; + } + + /** {@inheritDoc} */ + @Override + public ITestDevice getDevice() { + return mDevice; + } + + /** + * Test Entry Point + * + * <p>{@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + + initializeTest(listener); + + mTestRunHelper.startTest(1); + + try { + if (!verifyTestParameters()) { + return; + } + + // Stop logcat logging so we can record one logcat log per iteration + getDevice().stopLogcat(); + + // Run test iterations + for (int i = 0; i < mIterations; i++) { + CLog.i("---- Iteration " + i + " of " + (mIterations - 1) + " -----"); + + final ResultData d = new ResultData(); + d.setIteration(i); + Map<String, String> resultsDictionary = null; + resultsDictionary = runTest(d, getSingleTestTimeoutValue()); + + mLoopbackTestHelper.addTestData(d, resultsDictionary); + } + + mLoopbackTestHelper.processTestData(); + } finally { + Map<String, String> metrics = uploadLogsReturnMetrics(listener); + CLog.i("Uploading metrics values:\n" + Arrays.toString(metrics.entrySet().toArray())); + mTestRunHelper.endTest(metrics); + deleteAllTempFiles(); + getDevice().startLogcat(); + } + } + + private void initializeTest(ITestInvocationListener listener) + throws UnsupportedOperationException, DeviceNotAvailableException { + + mFileDataKeyMap = getLogFileDataKeyMap(); + TestIdentifier testId = new TestIdentifier(getClass().getCanonicalName(), mRunKey); + + // Allocate helpers + mTestRunHelper = new TestRunHelper(listener, testId); + mLoopbackTestHelper = new AudioLoopbackTestHelper(mIterations); + + getDevice().disableKeyguard(); + getDevice().waitForDeviceAvailable(DEVICE_SYNC_MS); + + getDevice().setDate(new Date()); + CLog.i("syncing device time to host time"); + } + + private Map<String, String> runTest(ResultData data, final long timeout) + throws DeviceNotAvailableException { + + // start measurement and wait for result file + final NullOutputReceiver receiver = new NullOutputReceiver(); + + final String loopbackCmd = getTestCommand(data.getIteration()); + CLog.i("Loopback cmd: " + loopbackCmd); + + // Clear logcat + // Seems like getDevice().clearLogcat(); doesn't do anything? + // Do it through ADB + getDevice().executeAdbCommand("logcat", "-c"); + final long deviceTestStartTime = getDevice().getDeviceDate(); + + getDevice() + .executeShellCommand( + loopbackCmd, receiver, TIMEOUT_MS, TimeUnit.MILLISECONDS, MAX_ATTEMPTS); + + final long loopbackStartTime = System.currentTimeMillis(); + File loopbackReport = null; + + data.setDeviceTestStartTime(deviceTestStartTime); + + // Try to retrieve result file from device. + final String resultFilename = getDeviceFilename(LogFileType.RESULT, data.getIteration()); + do { + RunUtil.getDefault().sleep(POLLING_INTERVAL_MS); + if (getDevice().doesFileExist(resultFilename)) { + // Store device log files in tmp directory on Host and add to ResultData object + storeDeviceFilesOnHost(data); + final String reportFilename = data.getLogFile(LogFileType.RESULT); + if (reportFilename != null && !reportFilename.isEmpty()) { + loopbackReport = new File(reportFilename); + if (loopbackReport.length() > 0) { + break; + } + } + } + + data.setIsTimedOut(System.currentTimeMillis() - loopbackStartTime >= timeout); + } while (!data.hasLogFile(LogFileType.RESULT) && !data.isTimedOut()); + + // Grab logcat for iteration + final InputStreamSource lc = getDevice().getLogcatSince(deviceTestStartTime); + saveLogcatForIteration(data, lc, data.getIteration()); + + // Check if test timed out. If so, don't fail the test, but return to upper logic. + // We accept certain number of individual test timeouts. + if (data.isTimedOut()) { + // No device result files retrieved, so no need to parse + return null; + } + + // parse result + Map<String, String> loopbackResult = null; + + try { + loopbackResult = + AudioLoopbackTestHelper.parseKeyValuePairFromFile( + loopbackReport, METRICS_KEY_MAP, mKeyPrefix, "=", "%s: %s"); + populateResultData(loopbackResult, data); + + // Trust but verify, so get Audio Level from ADB and compare to value from app + final int adbAudioLevel = + AudioLevelUtility.extractDeviceAudioLevelFromAdbShell(getDevice()); + if (data.getAudioLevel() != adbAudioLevel) { + final String errMsg = + String.format( + "App Audio Level (%1$d)differs from ADB level (%2$d)", + data.getAudioLevel(), adbAudioLevel); + mTestRunHelper.reportFailure(errMsg); + } + } catch (final IOException ioe) { + CLog.e(ioe); + mTestRunHelper.reportFailure("I/O error while parsing Loopback result."); + } catch (final NumberFormatException ne) { + CLog.e(ne); + mTestRunHelper.reportFailure("Number format error parsing Loopback result."); + } + + return loopbackResult; + } + + private String getMetricsKey(final String key) { + return mKeyPrefix + key; + } + private final long getSingleTestTimeoutValue() { + return Long.parseLong(mBufferTestDuration) * 1000 + TIMEOUT_MS; + } + + private Map<String, String> uploadLogsReturnMetrics(ITestInvocationListener listener) + throws DeviceNotAvailableException { + + // "resultDictionary" is used to post results to dashboards like BlackBox + // "results" contains test logs to be uploaded; i.e. to Sponge + + List<ResultData> results = null; + Map<String, String> resultDictionary = new HashMap<String, String>(); + + switch (getTestType()) { + case GLITCH: + resultDictionary = mLoopbackTestHelper.getResultDictionaryForIteration(0); + // Upload all test files to be backward compatible with old test + results = mLoopbackTestHelper.getAllTestData(); + break; + case LATENCY: + { + final int nrOfValidResults = mLoopbackTestHelper.processTestData(); + if (nrOfValidResults == 0) { + mTestRunHelper.reportFailure("No good data was collected"); + } else { + // use dictionary collected from single test run + resultDictionary = mLoopbackTestHelper.getResultDictionaryForIteration(0); + } + + // Upload all test files to be backward compatible with old test + results = mLoopbackTestHelper.getAllTestData(); + } + break; + case LATENCY_STRESS: + { + final int nrOfValidResults = mLoopbackTestHelper.processTestData(); + if (nrOfValidResults == 0) { + mTestRunHelper.reportFailure("No good data was collected"); + } else { + mLoopbackTestHelper.populateStressTestMetrics(resultDictionary, mKeyPrefix); + } + + results = mLoopbackTestHelper.getWorstResults(MAX_NR_OF_LOG_UPLOADS); + + // Save all test data in a spreadsheet style csv file for post test analysis + try { + saveResultsAsCSVFile(listener); + } catch (final IOException e) { + CLog.e(e); + } + } + break; + default: + break; + } + + // Upload relevant logs + for (final ResultData d : results) { + final LogFileType[] logFileTypes = getLogFileTypesForCurrentTest(); + for (final LogFileType logType : logFileTypes) { + uploadLog(listener, logType, d); + } + } + + return resultDictionary; + } + + private TestType getTestType() { + if (mTestType.equals(TESTTYPE_GLITCH_STR)) { + if (GLITCH_ITERATIONS_LOWER_BOUND <= mIterations + && mIterations <= GLITCH_ITERATIONS_UPPER_BOUND) { + return TestType.GLITCH; + } + } + + if (mTestType.equals(TESTTYPE_LATENCY_STR)) { + if (mIterations == 1) { + return TestType.LATENCY; + } + + if (LATENCY_ITERATIONS_LOWER_BOUND <= mIterations + && mIterations <= LATENCY_ITERATIONS_UPPER_BOUND) { + return TestType.LATENCY_STRESS; + } + } + + return TestType.NONE; + } + + private boolean verifyTestParameters() { + if (getTestType() != TestType.NONE) { + return true; + } + + if (mTestType.equals(TESTTYPE_GLITCH_STR) + && (mIterations < GLITCH_ITERATIONS_LOWER_BOUND + || mIterations > GLITCH_ITERATIONS_UPPER_BOUND)) { + final String error = + String.format( + ERR_PARAMETER_OUT_OF_BOUNDS, + "iterations", + GLITCH_ITERATIONS_LOWER_BOUND, + GLITCH_ITERATIONS_UPPER_BOUND); + mTestRunHelper.reportFailure(error); + return false; + } + + if (mTestType.equals(TESTTYPE_LATENCY_STR) + && (mIterations < LATENCY_ITERATIONS_LOWER_BOUND + || mIterations > LATENCY_ITERATIONS_UPPER_BOUND)) { + final String error = + String.format( + ERR_PARAMETER_OUT_OF_BOUNDS, + "iterations", + LATENCY_ITERATIONS_LOWER_BOUND, + LATENCY_ITERATIONS_UPPER_BOUND); + mTestRunHelper.reportFailure(error); + return false; + } + + return true; + } + + private void populateResultData(final Map<String, String> results, ResultData data) { + if (results == null || results.isEmpty()) { + return; + } + + String key = getMetricsKey(KEY_RESULT_LATENCY_MS); + if (results.containsKey(key)) { + data.setLatency(Float.parseFloat(results.get(key))); + } + + key = getMetricsKey(KEY_RESULT_LATENCY_CONFIDENCE); + if (results.containsKey(key)) { + data.setConfidence(Float.parseFloat(results.get(key))); + } + + key = getMetricsKey(KEY_RESULT_AUDIO_LEVEL); + if (results.containsKey(key)) { + data.setAudioLevel(Integer.parseInt(results.get(key))); + } + + key = getMetricsKey(KEY_RESULT_RMS); + if (results.containsKey(key)) { + data.setRMS(Float.parseFloat(results.get(key))); + } + + key = getMetricsKey(KEY_RESULT_RMS_AVERAGE); + if (results.containsKey(key)) { + data.setRMSAverage(Float.parseFloat(results.get(key))); + } + + key = getMetricsKey(KEY_RESULT_PERIOD_CONFIDENCE); + if (results.containsKey(key)) { + data.setPeriodConfidence(Float.parseFloat(results.get(key))); + } + + key = getMetricsKey(KEY_RESULT_SAMPLING_BLOCK_SIZE); + if (results.containsKey(key)) { + data.setBlockSize(Integer.parseInt(results.get(key))); + } + } + + private void storeDeviceFilesOnHost(ResultData data) throws DeviceNotAvailableException { + final int iteration = data.getIteration(); + for (final LogFileType log : getLogFileTypesForCurrentTest()) { + if (getDevice().doesFileExist(getDeviceFilename(log, iteration))) { + final String deviceFileName = getDeviceFilename(log, iteration); + final File logFile = getDevice().pullFile(deviceFileName); + data.setLogFile(log, logFile.getAbsolutePath()); + CLog.i("Delete file from device: " + deviceFileName); + deleteFileFromDevice(deviceFileName); + } + } + } + + private void deleteAllTempFiles() { + for (final ResultData d : mLoopbackTestHelper.getAllTestData()) { + final LogFileType[] logFileTypes = getLogFileTypesForCurrentTest(); + for (final LogFileType logType : logFileTypes) { + final String logFilename = d.getLogFile(logType); + if (logFilename == null || logFilename.isEmpty()) { + CLog.e("Logfile not found for LogFileType=" + logType.name()); + } else { + FileUtil.deleteFile(new File(logFilename)); + } + } + } + } + + private void deleteFileFromDevice(String deviceFileName) throws DeviceNotAvailableException { + getDevice().executeShellCommand("rm -f " + deviceFileName); + } + + private final LogFileType[] getLogFileTypesForCurrentTest() { + switch (getTestType()) { + case GLITCH: + return GLITCH_TEST_LOGS; + case LATENCY: + case LATENCY_STRESS: + return LATENCY_TEST_LOGS; + default: + return null; + } + } + + private String getKeyPrefixForIteration(int iteration) { + if (mIterations == 1) { + // If only one run, skip the iteration number + return mKeyPrefix; + } + return mKeyPrefix + iteration + "_"; + } + + private String getDeviceFilename(LogFileType key, int iteration) { + final Map<LogFileType, LogFileData> map = getLogFileDataKeyMap(); + if (map.containsKey(key)) { + final LogFileData data = map.get(key); + return String.format(FMT_DEVICE_PATH, iteration, data.fileExtension); + } + return null; + } + + private void uploadLog(ITestInvocationListener listener, LogFileType key, ResultData data) { + final Map<LogFileType, LogFileData> map = getLogFileDataKeyMap(); + if (!map.containsKey(key)) { + return; + } + + final LogFileData logInfo = map.get(key); + final String prefix = getKeyPrefixForIteration(data.getIteration()) + logInfo.filePrefix; + final LogDataType logDataType = logInfo.logDataType; + final String logFilename = data.getLogFile(key); + if (logFilename == null || logFilename.isEmpty()) { + CLog.e("Logfile not found for LogFileType=" + key.name()); + } else { + File logFile = new File(logFilename); + InputStreamSource iss = new FileInputStreamSource(logFile); + listener.testLog(prefix, logDataType, iss); + + // cleanup + iss.cancel(); + } + } + + private void saveLogcatForIteration(ResultData data, InputStreamSource logcat, int iteration) { + if (logcat == null) { + CLog.i("Logcat could not be saved for iteration " + iteration); + return; + } + + //create a temp file + File temp; + try { + temp = FileUtil.createTempFile("logcat_" + iteration + "_", ".txt"); + data.setLogFile(LogFileType.LOGCAT, temp.getAbsolutePath()); + + // Copy logcat data into temp file + Files.copy(logcat.createInputStream(), temp.toPath(), REPLACE_EXISTING); + logcat.cancel(); + } catch (final IOException e) { + CLog.i("Error when saving logcat for iteration=" + iteration); + CLog.e(e); + } + } + + private void saveResultsAsCSVFile(ITestInvocationListener listener) + throws DeviceNotAvailableException, IOException { + final File csvTmpFile = File.createTempFile("audio_test_data", "csv"); + mLoopbackTestHelper.writeAllResultsToCSVFile(csvTmpFile, getDevice()); + InputStreamSource iss = new FileInputStreamSource(csvTmpFile); + listener.testLog("audio_test_data", LogDataType.JACOCO_CSV, iss); + + // cleanup + iss.cancel(); + csvTmpFile.delete(); + } + + private String getTestCommand(int currentIteration) { + return String.format( + AM_CMD, + mSamplingFreq, + String.format(FMT_OUTPUT_PREFIX, currentIteration), + mMicSource, + mAudioThread, + mAudioLevel, + mTestType, + mBufferTestDuration); + } +} diff --git a/src/com/android/media/tests/AudioLoopbackTestHelper.java b/src/com/android/media/tests/AudioLoopbackTestHelper.java new file mode 100644 index 0000000..4e9c2b0 --- /dev/null +++ b/src/com/android/media/tests/AudioLoopbackTestHelper.java @@ -0,0 +1,608 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +import com.android.media.tests.AudioLoopbackImageAnalyzer.Result; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.util.Pair; + +import com.google.common.io.Files; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Helper class for AudioLoopbackTest. It keeps runtime data, analytics, */ +public class AudioLoopbackTestHelper { + + private StatisticsData mLatencyStats = null; + private StatisticsData mConfidenceStats = null; + private ArrayList<ResultData> mAllResults; + private ArrayList<ResultData> mGoodResults = new ArrayList<ResultData>(); + private ArrayList<ResultData> mBadResults = new ArrayList<ResultData>(); + private ArrayList<Map<String, String>> mResultDictionaries = + new ArrayList<Map<String, String>>(); + + // Controls acceptable tolerance in ms around median latency + private static final double TOLERANCE = 2.0; + + //=================================================================== + // ENUMS + //=================================================================== + public enum LogFileType { + RESULT, + WAVE, + GRAPH, + PLAYER_BUFFER, + PLAYER_BUFFER_HISTOGRAM, + PLAYER_BUFFER_PERIOD_TIMES, + RECORDER_BUFFER, + RECORDER_BUFFER_HISTOGRAM, + RECORDER_BUFFER_PERIOD_TIMES, + GLITCHES_MILLIS, + HEAT_MAP, + LOGCAT + } + + //=================================================================== + // INNER CLASSES + //=================================================================== + private class StatisticsData { + double mMin = 0; + double mMax = 0; + double mMean = 0; + double mMedian = 0; + + @Override + public String toString() { + return String.format( + "min = %1$f, max = %2$f, median=%3$f, mean = %4$f", mMin, mMax, mMedian, mMean); + } + } + + /** ResultData is an inner class that holds results and logfile info from each test run */ + public static class ResultData { + private Float mLatencyMs; + private Float mLatencyConfidence; + private Integer mAudioLevel; + private Integer mIteration; + private Long mDeviceTestStartTime; + private boolean mIsTimedOut = false; + private HashMap<LogFileType, String> mLogs = new HashMap<LogFileType, String>(); + private Result mImageAnalyzerResult = Result.UNKNOWN; + private String mFailureReason = null; + + // Optional + private Float mPeriodConfidence = Float.valueOf(0.0f); + private Float mRms = Float.valueOf(0.0f); + private Float mRmsAverage = Float.valueOf(0.0f); + private Integer mBblockSize = Integer.valueOf(0); + + public float getLatency() { + return mLatencyMs.floatValue(); + } + + public void setLatency(float latencyMs) { + this.mLatencyMs = Float.valueOf(latencyMs); + } + + public float getConfidence() { + return mLatencyConfidence.floatValue(); + } + + public void setConfidence(float latencyConfidence) { + this.mLatencyConfidence = Float.valueOf(latencyConfidence); + } + + public float getPeriodConfidence() { + return mPeriodConfidence.floatValue(); + } + + public void setPeriodConfidence(float periodConfidence) { + this.mPeriodConfidence = Float.valueOf(periodConfidence); + } + + public float getRMS() { + return mRms.floatValue(); + } + + public void setRMS(float rms) { + this.mRms = Float.valueOf(rms); + } + + public float getRMSAverage() { + return mRmsAverage.floatValue(); + } + + public void setRMSAverage(float rmsAverage) { + this.mRmsAverage = Float.valueOf(rmsAverage); + } + + public int getAudioLevel() { + return mAudioLevel.intValue(); + } + + public void setAudioLevel(int audioLevel) { + this.mAudioLevel = Integer.valueOf(audioLevel); + } + + public int getBlockSize() { + return mBblockSize.intValue(); + } + + public void setBlockSize(int blockSize) { + this.mBblockSize = Integer.valueOf(blockSize); + } + + public int getIteration() { + return mIteration.intValue(); + } + + public void setIteration(int iteration) { + this.mIteration = Integer.valueOf(iteration); + } + + public long getDeviceTestStartTime() { + return mDeviceTestStartTime.longValue(); + } + + public void setDeviceTestStartTime(long deviceTestStartTime) { + this.mDeviceTestStartTime = Long.valueOf(deviceTestStartTime); + } + + public Result getImageAnalyzerResult() { + return mImageAnalyzerResult; + } + + public void setImageAnalyzerResult(Result imageAnalyzerResult) { + this.mImageAnalyzerResult = imageAnalyzerResult; + } + + public String getFailureReason() { + return mFailureReason; + } + + public void setFailureReason(String failureReason) { + this.mFailureReason = failureReason; + } + + public boolean isTimedOut() { + return mIsTimedOut; + } + + public void setIsTimedOut(boolean isTimedOut) { + this.mIsTimedOut = isTimedOut; + } + + public String getLogFile(LogFileType log) { + return mLogs.get(log); + } + + public void setLogFile(LogFileType log, String filename) { + CLog.i("setLogFile: type=" + log.name() + ", filename=" + filename); + if (!mLogs.containsKey(log) && filename != null && !filename.isEmpty()) { + mLogs.put(log, filename); + } + } + + public boolean hasBadResults() { + return hasTimedOut() + || hasNoTestResults() + || hasNoLatencyResult() + || hasNoLatencyConfidence() + || mImageAnalyzerResult == Result.FAIL; + } + + public boolean hasTimedOut() { + return mIsTimedOut; + } + + public boolean hasLogFile(LogFileType log) { + return mLogs.containsKey(log); + } + + public boolean hasNoLatencyResult() { + return mLatencyMs == null; + } + + public boolean hasNoLatencyConfidence() { + return mLatencyConfidence == null; + } + + public boolean hasNoTestResults() { + return hasNoLatencyConfidence() && hasNoLatencyResult(); + } + + public static Comparator<ResultData> latencyComparator = + new Comparator<ResultData>() { + @Override + public int compare(ResultData o1, ResultData o2) { + return o1.mLatencyMs.compareTo(o2.mLatencyMs); + } + }; + + public static Comparator<ResultData> confidenceComparator = + new Comparator<ResultData>() { + @Override + public int compare(ResultData o1, ResultData o2) { + return o1.mLatencyConfidence.compareTo(o2.mLatencyConfidence); + } + }; + + public static Comparator<ResultData> iteratorComparator = + new Comparator<ResultData>() { + @Override + public int compare(ResultData o1, ResultData o2) { + return Integer.compare(o1.mIteration, o2.mIteration); + } + }; + + @Override + public String toString() { + final String NL = "\n"; + final StringBuilder sb = new StringBuilder(512); + sb.append("{").append(NL); + sb.append("{\nlatencyMs=").append(mLatencyMs).append(NL); + sb.append("latencyConfidence=").append(mLatencyConfidence).append(NL); + sb.append("isTimedOut=").append(mIsTimedOut).append(NL); + sb.append("iteration=").append(mIteration).append(NL); + sb.append("logs=").append(Arrays.toString(mLogs.values().toArray())).append(NL); + sb.append("audioLevel=").append(mAudioLevel).append(NL); + sb.append("deviceTestStartTime=").append(mDeviceTestStartTime).append(NL); + sb.append("rms=").append(mRms).append(NL); + sb.append("rmsAverage=").append(mRmsAverage).append(NL); + sb.append("}").append(NL); + return sb.toString(); + } + } + + public AudioLoopbackTestHelper(int iterations) { + mAllResults = new ArrayList<ResultData>(iterations); + } + + public void addTestData(ResultData data, Map<String, String> resultDictionary) { + mResultDictionaries.add(data.getIteration(), resultDictionary); + mAllResults.add(data); + + // Analyze captured screenshot to see if wave form is within reason + final String screenshot = data.getLogFile(LogFileType.GRAPH); + final Pair<Result, String> result = AudioLoopbackImageAnalyzer.analyzeImage(screenshot); + data.setImageAnalyzerResult(result.first); + data.setFailureReason(result.second); + } + + public final List<ResultData> getAllTestData() { + return mAllResults; + } + + public Map<String, String> getResultDictionaryForIteration(int i) { + return mResultDictionaries.get(i); + } + + /** + * Returns a list of the worst test result objects, up to maxNrOfWorstResults + * + * <p> + * + * <ol> + * <li> Tests in the bad results list are added first + * <li> If still space, add test results based on low confidence and then tests that are + * outside tolerance boundaries + * </ol> + * + * @param maxNrOfWorstResults + * @return list of worst test result objects + */ + public List<ResultData> getWorstResults(int maxNrOfWorstResults) { + int counter = 0; + final ArrayList<ResultData> worstResults = new ArrayList<ResultData>(maxNrOfWorstResults); + + for (final ResultData data : mBadResults) { + if (counter < maxNrOfWorstResults) { + worstResults.add(data); + counter++; + } + } + + for (final ResultData data : mGoodResults) { + if (counter < maxNrOfWorstResults) { + boolean failed = false; + if (data.getConfidence() < 1.0f) { + data.setFailureReason("Low confidence"); + failed = true; + } else if (data.getLatency() < (mLatencyStats.mMedian - TOLERANCE) + || data.getLatency() > (mLatencyStats.mMedian + TOLERANCE)) { + data.setFailureReason("Latency not within tolerance from median"); + failed = true; + } + + if (failed) { + worstResults.add(data); + counter++; + } + } + } + + return worstResults; + } + + public static Map<String, String> parseKeyValuePairFromFile( + File result, + final Map<String, String> dictionary, + final String resultKeyPrefix, + final String splitOn, + final String keyValueFormat) + throws IOException { + + final Map<String, String> resultMap = new HashMap<String, String>(); + final BufferedReader br = Files.newReader(result, StandardCharsets.UTF_8); + + try { + String line = br.readLine(); + while (line != null) { + line = line.trim().replaceAll(" +", " "); + final String[] tokens = line.split(splitOn); + if (tokens.length >= 2) { + final String key = tokens[0].trim(); + final String value = tokens[1].trim(); + if (dictionary.containsKey(key)) { + CLog.i(String.format(keyValueFormat, key, value)); + resultMap.put(resultKeyPrefix + dictionary.get(key), value); + } + } + line = br.readLine(); + } + } finally { + br.close(); + } + return resultMap; + } + + public int processTestData() { + + // Collect statistics about the test run + int nrOfValidResults = 0; + double sumLatency = 0; + double sumConfidence = 0; + + final int totalNrOfTests = mAllResults.size(); + mLatencyStats = new StatisticsData(); + mConfidenceStats = new StatisticsData(); + mBadResults = new ArrayList<ResultData>(); + mGoodResults = new ArrayList<ResultData>(totalNrOfTests); + + // Copy all results into Good results list + mGoodResults.addAll(mAllResults); + + for (final ResultData data : mAllResults) { + if (data.hasBadResults()) { + mBadResults.add(data); + continue; + } + // Get mean values + sumLatency += data.getLatency(); + sumConfidence += data.getConfidence(); + } + + if (!mBadResults.isEmpty()) { + analyzeBadResults(mBadResults, mAllResults.size()); + } + + // Remove bad runs from result array + mGoodResults.removeAll(mBadResults); + + // Fail test immediately if we don't have ANY good results + if (mGoodResults.isEmpty()) { + return 0; + } + + nrOfValidResults = mGoodResults.size(); + + // ---- LATENCY: Get Median, Min and Max values ---- + Collections.sort(mGoodResults, ResultData.latencyComparator); + + mLatencyStats.mMin = mGoodResults.get(0).mLatencyMs; + mLatencyStats.mMax = mGoodResults.get(nrOfValidResults - 1).mLatencyMs; + mLatencyStats.mMean = sumLatency / nrOfValidResults; + // Is array even or odd numbered + if (nrOfValidResults % 2 == 0) { + final int middle = nrOfValidResults / 2; + final float middleLeft = mGoodResults.get(middle - 1).mLatencyMs; + final float middleRight = mGoodResults.get(middle).mLatencyMs; + mLatencyStats.mMedian = (middleLeft + middleRight) / 2.0f; + } else { + // It's and odd numbered array, just grab the middle value + mLatencyStats.mMedian = mGoodResults.get(nrOfValidResults / 2).mLatencyMs; + } + + // ---- CONFIDENCE: Get Median, Min and Max values ---- + Collections.sort(mGoodResults, ResultData.confidenceComparator); + + mConfidenceStats.mMin = mGoodResults.get(0).mLatencyConfidence; + mConfidenceStats.mMax = mGoodResults.get(nrOfValidResults - 1).mLatencyConfidence; + mConfidenceStats.mMean = sumConfidence / nrOfValidResults; + // Is array even or odd numbered + if (nrOfValidResults % 2 == 0) { + final int middle = nrOfValidResults / 2; + final float middleLeft = mGoodResults.get(middle - 1).mLatencyConfidence; + final float middleRight = mGoodResults.get(middle).mLatencyConfidence; + mConfidenceStats.mMedian = (middleLeft + middleRight) / 2.0f; + } else { + // It's and odd numbered array, just grab the middle value + mConfidenceStats.mMedian = mGoodResults.get(nrOfValidResults / 2).mLatencyConfidence; + } + + for (final ResultData data : mGoodResults) { + // Check if within Latency Tolerance + if (data.getConfidence() < 1.0f) { + data.setFailureReason("Low confidence"); + } else if (data.getLatency() < (mLatencyStats.mMedian - TOLERANCE) + || data.getLatency() > (mLatencyStats.mMedian + TOLERANCE)) { + data.setFailureReason("Latency not within tolerance from median"); + } + } + + // Create histogram + // Strategy: Create buckets based on whole ints, like 16 ms, 17 ms, 18 ms etc. Count how + // many tests fall into each bucket. Just cast the float to an int, no rounding up/down + // required. + final int[] histogram = new int[(int) mLatencyStats.mMax + 1]; + for (final ResultData rd : mGoodResults) { + // Increase value in bucket + histogram[(int) (rd.mLatencyMs.floatValue())]++; + } + + CLog.i("========== VALID RESULTS ============================================"); + CLog.i(String.format("Valid tests: %1$d of %2$d", nrOfValidResults, totalNrOfTests)); + CLog.i("Latency: " + mLatencyStats.toString()); + CLog.i("Confidence: " + mConfidenceStats.toString()); + CLog.i("========== HISTOGRAM ================================================"); + for (int i = 0; i < histogram.length; i++) { + if (histogram[i] > 0) { + CLog.i(String.format("%1$01d ms => %2$d", i, histogram[i])); + } + } + + // VERIFY the good results by running image analysis on the + // screenshot of the incoming audio waveform + + return nrOfValidResults; + } + + public void writeAllResultsToCSVFile(File csvFile, ITestDevice device) + throws DeviceNotAvailableException, FileNotFoundException, + UnsupportedEncodingException { + + final String deviceType = device.getProperty("ro.build.product"); + final String buildId = device.getBuildAlias(); + final String serialNumber = device.getSerialNumber(); + + // Sort data on iteration + Collections.sort(mAllResults, ResultData.iteratorComparator); + + final StringBuilder sb = new StringBuilder(256); + final PrintWriter writer = new PrintWriter(csvFile, StandardCharsets.UTF_8.name()); + final String SEPARATOR = ","; + + // Write column labels + writer.println( + "Device Time,Device Type,Build Id,Serial Number,Iteration,Latency," + + "Confidence,Period Confidence,Block Size,Audio Level,RMS,RMS Average," + + "Image Analysis,Failure Reason"); + for (final ResultData data : mAllResults) { + final Instant instant = Instant.ofEpochSecond(data.mDeviceTestStartTime); + + sb.append(instant).append(SEPARATOR); + sb.append(deviceType).append(SEPARATOR); + sb.append(buildId).append(SEPARATOR); + sb.append(serialNumber).append(SEPARATOR); + sb.append(data.getIteration()).append(SEPARATOR); + sb.append(data.getLatency()).append(SEPARATOR); + sb.append(data.getConfidence()).append(SEPARATOR); + sb.append(data.getPeriodConfidence()).append(SEPARATOR); + sb.append(data.getBlockSize()).append(SEPARATOR); + sb.append(data.getAudioLevel()).append(SEPARATOR); + sb.append(data.getRMS()).append(SEPARATOR); + sb.append(data.getRMSAverage()).append(SEPARATOR); + sb.append(data.getImageAnalyzerResult().name()).append(SEPARATOR); + sb.append(data.getFailureReason()); + + writer.println(sb.toString()); + + sb.setLength(0); + } + writer.close(); + } + + private void analyzeBadResults(ArrayList<ResultData> badResults, int totalNrOfTests) { + int testNoData = 0; + int testTimeoutCounts = 0; + int testResultsNotFoundCounts = 0; + int testWithoutLatencyResultCount = 0; + int testWithoutConfidenceResultCount = 0; + + for (final ResultData data : badResults) { + if (data.hasTimedOut()) { + testTimeoutCounts++; + testNoData++; + continue; + } + + if (data.hasNoTestResults()) { + testResultsNotFoundCounts++; + testNoData++; + continue; + } + + if (data.hasNoLatencyResult()) { + testWithoutLatencyResultCount++; + testNoData++; + continue; + } + + if (data.hasNoLatencyConfidence()) { + testWithoutConfidenceResultCount++; + testNoData++; + continue; + } + } + + CLog.i("========== BAD RESULTS ============================================"); + CLog.i(String.format("No Data: %1$d of %2$d", testNoData, totalNrOfTests)); + CLog.i(String.format("Timed out: %1$d of %2$d", testTimeoutCounts, totalNrOfTests)); + CLog.i( + String.format( + "No results: %1$d of %2$d", testResultsNotFoundCounts, totalNrOfTests)); + CLog.i( + String.format( + "No Latency results: %1$d of %2$d", + testWithoutLatencyResultCount, totalNrOfTests)); + CLog.i( + String.format( + "No Confidence results: %1$d of %2$d", + testWithoutConfidenceResultCount, totalNrOfTests)); + } + + /** Generates metrics dictionary for stress test */ + public void populateStressTestMetrics( + Map<String, String> metrics, final String resultKeyPrefix) { + metrics.put(resultKeyPrefix + "total_nr_of_tests", Integer.toString(mAllResults.size())); + metrics.put(resultKeyPrefix + "nr_of_good_tests", Integer.toString(mGoodResults.size())); + metrics.put(resultKeyPrefix + "latency_max", Double.toString(mLatencyStats.mMax)); + metrics.put(resultKeyPrefix + "latency_min", Double.toString(mLatencyStats.mMin)); + metrics.put(resultKeyPrefix + "latency_mean", Double.toString(mLatencyStats.mMean)); + metrics.put(resultKeyPrefix + "latency_median", Double.toString(mLatencyStats.mMedian)); + metrics.put(resultKeyPrefix + "confidence_max", Double.toString(mConfidenceStats.mMax)); + metrics.put(resultKeyPrefix + "confidence_min", Double.toString(mConfidenceStats.mMin)); + metrics.put(resultKeyPrefix + "confidence_mean", Double.toString(mConfidenceStats.mMean)); + metrics.put( + resultKeyPrefix + "confidence_median", Double.toString(mConfidenceStats.mMedian)); + } +} diff --git a/src/com/android/media/tests/Camera2FrameworkStressTest.java b/src/com/android/media/tests/Camera2FrameworkStressTest.java new file mode 100644 index 0000000..8d62595 --- /dev/null +++ b/src/com/android/media/tests/Camera2FrameworkStressTest.java @@ -0,0 +1,188 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.IFileEntry; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.util.FileUtil; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Camera2 framework stress test + * This is a test invocation that runs stress tests against Camera2 framework (API & HAL) to + * isolate stability issues from Camera application. + * This invocation uses a Camera2InstrumentationTestRunner to run a set of + * Camera framework stress tests. + */ +@OptionClass(alias = "camera2-framework-stress") +public class Camera2FrameworkStressTest extends CameraTestBase { + + // Keys in instrumentation test metrics + private static final String RESULT_DIR = "/sdcard/camera-out/"; + private static final String RESULT_FILE_FORMAT = RESULT_DIR + "fwk-stress_camera_%s.txt"; + private static final Pattern RESULT_FILE_REGEX = Pattern.compile( + "^fwk-stress_camera_(?<id>.+).txt"); + private static final String KEY_NUM_ATTEMPTS = "numAttempts"; + private static final String KEY_ITERATION = "iteration"; + + public Camera2FrameworkStressTest() { + // Note that default value in constructor will be overridden by the passing option from + // a command line. + setTestPackage("com.android.mediaframeworktest"); + setTestRunner("com.android.mediaframeworktest.Camera2InstrumentationTestRunner"); + setRuKey("CameraFrameworkStress"); + setTestTimeoutMs(2 * 60 * 60 * 1000); // 2 hours + setLogcatOnFailure(true); + } + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + runInstrumentationTest(listener, new CollectingListener(listener)); + } + + /** + * A listener to collect the output from test run and fatal errors + */ + private class CollectingListener extends DefaultCollectingListener { + + public CollectingListener(ITestInvocationListener listener) { + super(listener); + } + + @Override + public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) { + if (testMetrics == null) { + return; // No-op if there is nothing to post. + } + for (Map.Entry<String, String> metric : testMetrics.entrySet()) { + getAggregatedMetrics().put(metric.getKey(), metric.getValue()); + } + } + + @Override + public void testEnded(TestIdentifier test, long endTime, Map<String, String> testMetrics) { + if (hasTestRunFatalError()) { + CLog.v("The instrumentation result not found. Fall back to get the metrics from a " + + "log file. errorMsg: %s", getCollectingListener().getErrorMessage()); + } + + // For stress test, parse the metrics from a log file. + testMetrics = parseLog(test.getTestName()); + super.testEnded(test, endTime, testMetrics); + } + + // Return null if failed to parse the result file or the test didn't even start. + private Map<String, String> parseLog(String testName) { + Map<String, String> postMetrics = new HashMap<String, String>(); + String resultFilePath; + try { + for (String cameraId : getCameraIdList(RESULT_DIR)) { + File outputFile = FileUtil.createTempFile("fwk-stress", ".txt"); + resultFilePath = getResultFilePath(cameraId); + getDevice().pullFile(resultFilePath, outputFile); + if (outputFile == null) { + throw new DeviceNotAvailableException(String.format("Failed to pull the " + + "result file: %s", resultFilePath), + getDevice().getSerialNumber()); + } + + BufferedReader reader = new BufferedReader(new FileReader(outputFile)); + String line; + Map<String, String> resultMap = new HashMap<String, String>(); + + // Parse results from log file that contain the key-value pairs. + // eg. "numAttempts=10|iteration=9[|cameraId=0]" + try { + while ((line = reader.readLine()) != null) { + String[] pairs = line.split("\\|"); + for (String pair : pairs) { + String[] keyValue = pair.split("="); + // Each should be a pair of key and value. + String key = keyValue[0].trim(); + String value = keyValue[1].trim(); + resultMap.put(key, value); + } + } + } finally { + reader.close(); + } + + // Fail if a stress test doesn't start. + if (0 == Integer.parseInt(resultMap.get(KEY_NUM_ATTEMPTS))) { + CLog.w("Failed to start stress tests. test setup configured incorrectly?"); + return null; + } + // Post the number of iterations only with the key name associated with + // test name and camera ID. + String keyName = testName + "_" + cameraId; + postMetrics.put(keyName, resultMap.get(KEY_ITERATION)); + } + } catch (IOException e) { + CLog.w("Couldn't parse the output log file"); + CLog.e(e); + } catch (DeviceNotAvailableException e) { + CLog.w("Could not pull file: %s, error:", RESULT_DIR); + CLog.e(e); + } catch (NumberFormatException e) { + CLog.w("Could not find the key in file: %s, error:", KEY_NUM_ATTEMPTS); + CLog.e(e); + } + return postMetrics; + } + } + + private ArrayList<String> getCameraIdList(String resultDir) throws DeviceNotAvailableException { + // The result files are created per each camera ID + ArrayList<String> cameraIds = new ArrayList<>(); + IFileEntry dirEntry = getDevice().getFileEntry(resultDir); + if (dirEntry != null) { + for (IFileEntry file : dirEntry.getChildren(false)) { + String fileName = file.getName(); + Matcher matcher = RESULT_FILE_REGEX.matcher(fileName); + if (matcher.matches()) { + cameraIds.add(matcher.group("id")); + } + } + } + + if (cameraIds.isEmpty()) { + CLog.w("No camera ID is found in %s. The resultToFile instrumentation argument is set" + + " to false?", resultDir); + } + return cameraIds; + } + + private String getResultFilePath(String cameraId) { + return String.format(RESULT_FILE_FORMAT, cameraId); + } +} diff --git a/src/com/android/media/tests/Camera2LatencyTest.java b/src/com/android/media/tests/Camera2LatencyTest.java new file mode 100644 index 0000000..5a06822 --- /dev/null +++ b/src/com/android/media/tests/Camera2LatencyTest.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Camera app latency test + * + * Runs Camera app latency test to measure Camera capture session time and reports the metrics. + */ +@OptionClass(alias = "camera-latency") +public class Camera2LatencyTest extends CameraTestBase { + + public Camera2LatencyTest() { + setTestPackage("com.google.android.camera"); + setTestClass("com.android.camera.latency.CameraCaptureSessionTest"); + setTestRunner("android.test.InstrumentationTestRunner"); + setRuKey("CameraAppLatency"); + setTestTimeoutMs(60 * 60 * 1000); // 1 hour + } + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + runInstrumentationTest(listener, new CollectingListener(listener)); + } + + /** + * A listener to collect the results from each test run, then forward them to other listeners. + */ + private class CollectingListener extends DefaultCollectingListener { + + public CollectingListener(ITestInvocationListener listener) { + super(listener); + } + + @Override + public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) { + // Test metrics accumulated will be posted at the end of test run. + getAggregatedMetrics().putAll(parseResults(test.getTestName(), testMetrics)); + } + + private Map<String, String> parseResults(String testName, Map<String, String> testMetrics) { + final Pattern STATS_REGEX = Pattern.compile( + "^(?<latency>[0-9.]+)\\|(?<values>[0-9 .-]+)"); + + // Parse activity time stats from the instrumentation result. + // Format : <metric_key>=<average_of_latency>|<raw_data> + // Example: + // FirstCaptureResultTimeMs=38|13 48 ... 35 + // SecondCaptureResultTimeMs=29.2|65 24 ... 0 + // CreateTimeMs=373.6|382 364 ... 323 + // + // Then report only the first two startup time of cold startup and average warm startup. + Map<String, String> parsed = new HashMap<String, String>(); + for (Map.Entry<String, String> metric : testMetrics.entrySet()) { + Matcher matcher = STATS_REGEX.matcher(metric.getValue()); + if (matcher.matches()) { + String keyName = String.format("%s_%s", testName, metric.getKey()); + parsed.put(keyName, matcher.group("latency")); + } else { + CLog.w(String.format("Stats not in correct format: %s", metric.getValue())); + } + } + return parsed; + } + } +} diff --git a/src/com/android/media/tests/Camera2StressTest.java b/src/com/android/media/tests/Camera2StressTest.java new file mode 100644 index 0000000..77868d9 --- /dev/null +++ b/src/com/android/media/tests/Camera2StressTest.java @@ -0,0 +1,188 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.IFileEntry; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.FileInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.util.FileUtil; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Camera2 stress test + * Since Camera stress test can drain the battery seriously. Need to split + * the test suite into separate test invocation for each test method. + * <p/> + */ +@OptionClass(alias = "camera2-stress") +public class Camera2StressTest extends CameraTestBase { + + private static final String RESULT_FILE = "/sdcard/camera-out/stress.txt"; + private static final String FAILURE_SCREENSHOT_DIR = "/sdcard/camera-screenshot/"; + private static final String KEY_NUM_ATTEMPTS = "numAttempts"; + private static final String KEY_ITERATION = "iteration"; + + public Camera2StressTest() { + setTestPackage("com.google.android.camera"); + setTestClass("com.android.camera.stress.CameraStressTest"); + setTestRunner("android.test.InstrumentationTestRunner"); + setRuKey("CameraAppStress"); + setTestTimeoutMs(6 * 60 * 60 * 1000); // 6 hours + setLogcatOnFailure(true); + } + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + runInstrumentationTest(listener, new CollectingListener(listener)); + } + + /** + * A listener to collect the output from test run and fatal errors + */ + private class CollectingListener extends DefaultCollectingListener { + + public CollectingListener(ITestInvocationListener listener) { + super(listener); + } + + @Override + public void testEnded(TestIdentifier test, long endTime, Map<String, String> testMetrics) { + if (hasTestRunFatalError()) { + CLog.v("The instrumentation result not found. Fall back to get the metrics from a " + + "log file. errorMsg: %s", getCollectingListener().getErrorMessage()); + } + // TODO: Will get the additional metrics to file to prevent result loss + + // Don't need to report the KEY_NUM_ATTEMPS to dashboard + // Iteration will be read from file + testMetrics.remove(KEY_NUM_ATTEMPTS); + testMetrics.remove(KEY_ITERATION); + + // add testMethod name to the metric + Map<String, String> namedTestMetrics = new HashMap<>(); + for (Entry<String, String> entry : testMetrics.entrySet()) { + namedTestMetrics.put(test.getTestName() + entry.getKey(), entry.getValue()); + } + + // parse the iterations metrics from the stress log files + parseLog(test.getTestName(), namedTestMetrics); + + postScreenshotOnFailure(test); + super.testEnded(test, endTime, namedTestMetrics); + } + + private void postScreenshotOnFailure(TestIdentifier test) { + File tmpDir = null; + try { + IFileEntry screenshotDir = getDevice().getFileEntry(FAILURE_SCREENSHOT_DIR); + if (screenshotDir != null && screenshotDir.isDirectory()) { + tmpDir = FileUtil.createTempDir("screenshot"); + for (IFileEntry remoteFile : screenshotDir.getChildren(false)) { + if (remoteFile.isDirectory()) { + continue; + } + if (!remoteFile.getName().contains(test.getTestName())) { + CLog.v("Skipping the screenshot (%s) that doesn't match the current " + + "test (%s)", remoteFile.getName(), test.getTestName()); + continue; + } + File screenshot = new File(tmpDir, remoteFile.getName()); + if (!getDevice().pullFile(remoteFile.getFullPath(), screenshot)) { + CLog.w("Could not pull screenshot: %s", remoteFile.getFullPath()); + continue; + } + testLog( + "screenshot_" + screenshot.getName(), + LogDataType.PNG, + new FileInputStreamSource(screenshot)); + } + } + } catch (DeviceNotAvailableException e) { + CLog.e(e); + } catch (IOException e) { + CLog.e(e); + } finally { + FileUtil.recursiveDelete(tmpDir); + } + } + + // Return null if failed to parse the result file or the test didn't even start. + private void parseLog(String testName, Map<String, String> testMetrics) { + try { + File outputFile = FileUtil.createTempFile("stress", ".txt"); + getDevice().pullFile(RESULT_FILE, outputFile); + if (outputFile == null) { + throw new DeviceNotAvailableException(String.format("Failed to pull the result" + + "file: %s", RESULT_FILE), getDevice().getSerialNumber()); + } + BufferedReader reader = new BufferedReader(new FileReader(outputFile)); + String line; + Map<String, String> resultMap = new HashMap<>(); + + // Parse results from log file that contain the key-value pairs. + // eg. "numAttempts=10|iteration=9" + try { + while ((line = reader.readLine()) != null) { + String[] pairs = line.split("\\|"); + for (String pair : pairs) { + String[] keyValue = pair.split("="); + // Each should be a pair of key and value. + String key = keyValue[0].trim(); + String value = keyValue[1].trim(); + resultMap.put(key, value); + CLog.v("%s: %s", key, value); + } + } + } finally { + reader.close(); + } + + // Fail if a stress test doesn't start. + if (0 == Integer.parseInt(resultMap.get(KEY_NUM_ATTEMPTS))) { + CLog.w("Failed to start stress tests. test setup configured incorrectly?"); + return; + } + // Post the number of iterations only with the test name as key. + testMetrics.put(testName, resultMap.get(KEY_ITERATION)); + } catch (IOException e) { + CLog.w("Couldn't parse the output log file:"); + CLog.e(e); + } catch (DeviceNotAvailableException e) { + CLog.w("Could not pull file: %s, error:", RESULT_FILE); + CLog.e(e); + } catch (NumberFormatException e) { + CLog.w("Could not find the key in file: %s, error:", KEY_NUM_ATTEMPTS); + CLog.e(e); + } + } + } +} diff --git a/src/com/android/media/tests/CameraBurstStartupTest.java b/src/com/android/media/tests/CameraBurstStartupTest.java new file mode 100644 index 0000000..74d7e87 --- /dev/null +++ b/src/com/android/media/tests/CameraBurstStartupTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Camera burst startup test + * + * <p>Runs Camera device performance test to measure time for taking a burst shot. + */ +@OptionClass(alias = "camera-burst-shot") +public class CameraBurstStartupTest extends CameraTestBase { + + private static final Pattern STATS_REGEX = Pattern.compile("^(?<average>[0-9.]+)"); + + public CameraBurstStartupTest() { + setTestPackage("com.google.android.camera"); + setTestClass("com.android.camera.latency.BurstStartupTest"); + setTestRunner("android.test.InstrumentationTestRunner"); + setRuKey("CameraBurstStartup"); + setTestTimeoutMs(60 * 60 * 1000); // 1 hour + } + + /** {@inheritDoc} */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + runInstrumentationTest(listener, new CollectingListener(listener)); + } + + /** A listener to collect the output from test run and fatal errors */ + private class CollectingListener extends DefaultCollectingListener { + + public CollectingListener(ITestInvocationListener listener) { + super(listener); + } + + @Override + public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) { + // Test metrics accumulated will be posted at the end of test run. + getAggregatedMetrics().putAll(parseResults(test.getTestName(), testMetrics)); + } + + public Map<String, String> parseResults(String testName, Map<String, String> testMetrics) { + // Parse burst startup stats from the instrumentation result. + Map<String, String> parsed = new HashMap<String, String>(); + for (Map.Entry<String, String> metric : testMetrics.entrySet()) { + Matcher matcher = STATS_REGEX.matcher(metric.getValue()); + + if (matcher.matches()) { + // Key name consists of a pair of test name and metric name. + String keyName = String.format("%s_%s", testName, metric.getKey()); + parsed.put(keyName, matcher.group("average")); + } else { + CLog.w(String.format("Stats not in correct format: %s", metric.getValue())); + } + } + return parsed; + } + } +} diff --git a/src/com/android/media/tests/CameraLatencyTest.java b/src/com/android/media/tests/CameraLatencyTest.java new file mode 100644 index 0000000..f44fd59 --- /dev/null +++ b/src/com/android/media/tests/CameraLatencyTest.java @@ -0,0 +1,369 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.IDevice; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.android.tradefed.config.Option; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.CollectingTestListener; +import com.android.tradefed.result.FileInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.RegexTrie; +import com.android.tradefed.util.StreamUtil; + +import junit.framework.TestCase; + +import org.junit.Assert; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Runs the Camera latency testcases. + * FIXME: more details + * <p/> + * Note that this test will not run properly unless /sdcard is mounted and writable. + */ +public class CameraLatencyTest implements IDeviceTest, IRemoteTest { + ITestDevice mTestDevice = null; + + // Constants for running the tests + private static final String TEST_PACKAGE_NAME = "com.google.android.camera.tests"; + + private final String mOutputPath = "mediaStressOut.txt"; + + //Max timeout for the test - 30 mins + private static final int MAX_TEST_TIMEOUT = 30 * 60 * 1000; + + /** + * Stores the test cases that we should consider running. + * + * <p>This currently consists of "startup" and "latency" + */ + private List<TestInfo> mTestCases = new ArrayList<>(); + + // Options for the running the gCam test + @Option(name = "gCam", description = "Run gCam startup test") + private boolean mGcam = false; + + + /** + * A struct that contains useful info about the tests to run + */ + static class TestInfo { + public String mTestName = null; + public String mClassName = null; + public String mTestMetricsName = null; + public RegexTrie<String> mPatternMap = new RegexTrie<>(); + + @Override + public String toString() { + return String.format("TestInfo: name(%s) class(%s) metric(%s) patterns(%s)", mTestName, + mClassName, mTestMetricsName, mPatternMap); + } + } + + /** + * Set up the configurations for the test cases we want to run + */ + private void testInfoSetup() { + // Startup tests + TestInfo t = new TestInfo(); + + if (mGcam) { + t.mTestName = "testLaunchCamera"; + t.mClassName = "com.android.camera.stress.CameraStartUp"; + t.mTestMetricsName = "GCameraStartup"; + RegexTrie<String> map = t.mPatternMap; + map = t.mPatternMap; + map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)"); + map.put("CameraStartup", "^Camera average startup time: (\\d+) ms"); + mTestCases.add(t); + } else { + t.mTestName = "startup"; + t.mClassName = "com.android.camera.stress.CameraStartUp"; + t.mTestMetricsName = "CameraVideoRecorderStartup"; + RegexTrie<String> map = t.mPatternMap; + map = t.mPatternMap; + map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)"); + map.put("CameraStartup", "^Camera average startup time: (\\d+) ms"); + map.put("FirstVideoStartup", "^First Video Startup: (\\d+)"); + map.put("VideoStartup", "^Video average startup time: (\\d+) ms"); + mTestCases.add(t); + + // Latency tests + t = new TestInfo(); + t.mTestName = "latency"; + t.mClassName = "com.android.camera.stress.CameraLatency"; + t.mTestMetricsName = "CameraLatency"; + map = t.mPatternMap; + map.put("AutoFocus", "^Avg AutoFocus = (\\d+)"); + map.put("ShutterLag", "^Avg mShutterLag = (\\d+)"); + map.put("Preview", "^Avg mShutterToPictureDisplayedTime = (\\d+)"); + map.put("RawPictureGeneration", "^Avg mPictureDisplayedToJpegCallbackTime = (\\d+)"); + map.put("GenTimeDiffOverJPEGAndRaw", "^Avg mJpegCallbackFinishTime = (\\d+)"); + map.put("FirstPreviewTime", "^Avg FirstPreviewTime = (\\d+)"); + mTestCases.add(t); + } + + } + + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + Assert.assertNotNull(mTestDevice); + testInfoSetup(); + for (TestInfo test : mTestCases) { + cleanTmpFiles(); + executeTest(test, listener); + logOutputFile(test, listener); + } + + cleanTmpFiles(); + } + + private void executeTest(TestInfo test, ITestInvocationListener listener) + throws DeviceNotAvailableException { + IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, + mTestDevice.getIDevice()); + CollectingTestListener auxListener = new CollectingTestListener(); + + runner.setClassName(test.mClassName); + runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS); + if (mGcam) { + runner.setMethodName(test.mClassName, test.mTestName); + } + mTestDevice.runInstrumentationTests(runner, listener, auxListener); + + // Grab a bugreport if warranted + if (auxListener.hasFailedTests()) { + CLog.i("Grabbing bugreport after test '%s' finished with %d failures.", + test.mTestName, auxListener.getNumAllFailedTests()); + InputStreamSource bugreport = mTestDevice.getBugreport(); + listener.testLog(String.format("bugreport-%s.txt", test.mTestName), + LogDataType.BUGREPORT, bugreport); + bugreport.cancel(); + } + } + + /** + * Clean up temp files from test runs + * <p /> + * Note that all photos on the test device will be removed + */ + private void cleanTmpFiles() throws DeviceNotAvailableException { + String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + //TODO: Remove the DCIM folder when the bug is fixed. + mTestDevice.executeShellCommand(String.format("rm %s/DCIM/Camera/*", extStore)); + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath)); + } + + /** + * Pull the output file from the device, add it to the logs, and also parse out the relevant + * test metrics and report them. + */ + private void logOutputFile(TestInfo test, ITestInvocationListener listener) + throws DeviceNotAvailableException { + File outputFile = null; + InputStreamSource outputSource = null; + try { + outputFile = mTestDevice.pullFileFromExternal(mOutputPath); + + if (outputFile == null) { + return; + } + + // Upload a verbatim copy of the output file + CLog.d("Sending %d byte file %s into the logosphere!", outputFile.length(), outputFile); + outputSource = new FileInputStreamSource(outputFile); + listener.testLog(String.format("output-%s.txt", test.mTestName), LogDataType.TEXT, + outputSource); + + // Parse the output file to upload aggregated metrics + parseOutputFile(test, new FileInputStream(outputFile), listener); + } catch (IOException e) { + CLog.e("IOException while reading or parsing output file"); + CLog.e(e); + } finally { + FileUtil.deleteFile(outputFile); + StreamUtil.cancel(outputSource); + } + } + + /** + * Parse the relevant metrics from the Instrumentation test output file + */ + private void parseOutputFile(TestInfo test, InputStream dataStream, + ITestInvocationListener listener) { + Map<String, String> runMetrics = new HashMap<>(); + + // try to parse it + String contents; + try { + contents = StreamUtil.getStringFromStream(dataStream); + } catch (IOException e) { + CLog.e("Got IOException during %s test processing", test.mTestName); + CLog.e(e); + return; + } + + List<String> lines = Arrays.asList(contents.split("\n")); + ListIterator<String> lineIter = lines.listIterator(); + String line; + while (lineIter.hasNext()) { + line = lineIter.next(); + List<List<String>> capture = new ArrayList<>(1); + String key = test.mPatternMap.retrieve(capture, line); + if (key != null) { + CLog.d("Got %s key '%s' and captures '%s'", test.mTestName, key, + capture.toString()); + } else if (line.isEmpty()) { + // ignore + continue; + } else { + CLog.d("Got unmatched line: %s", line); + continue; + } + + runMetrics.put(key, capture.get(0).get(0)); + } + + reportMetrics(listener, test, runMetrics); + } + + /** + * Report run metrics by creating an empty test run to stick them in + * <p /> + * Exposed for unit testing + */ + void reportMetrics(ITestInvocationListener listener, TestInfo test, + Map<String, String> metrics) { + // Create an empty testRun to report the parsed runMetrics + CLog.d("About to report metrics for %s: %s", test.mTestMetricsName, metrics); + listener.testRunStarted(test.mTestMetricsName, 0); + listener.testRunEnded(0, metrics); + } + + @Override + public void setDevice(ITestDevice device) { + mTestDevice = device; + } + + @Override + public ITestDevice getDevice() { + return mTestDevice; + } + + /** + * A meta-test to ensure that bits of the CameraLatencyTest are working properly + */ + public static class MetaTest extends TestCase { + private CameraLatencyTest mTestInstance = null; + + private TestInfo mTestInfo = null; + + private TestInfo mReportedTestInfo = null; + private Map<String, String> mReportedMetrics = null; + + private static String join(String... pieces) { + StringBuilder sb = new StringBuilder(); + for (String piece : pieces) { + sb.append(piece); + sb.append("\n"); + } + return sb.toString(); + } + + @Override + public void setUp() throws Exception { + mTestInstance = new CameraLatencyTest() { + @Override + void reportMetrics(ITestInvocationListener l, TestInfo test, + Map<String, String> metrics) { + mReportedTestInfo = test; + mReportedMetrics = metrics; + } + }; + + // Startup tests + mTestInfo = new TestInfo(); + TestInfo t = mTestInfo; // convenience alias + t.mTestName = "startup"; + t.mClassName = "com.android.camera.stress.CameraStartUp"; + t.mTestMetricsName = "camera_video_recorder_startup"; + RegexTrie<String> map = t.mPatternMap; + map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)"); + map.put("CameraStartup", "^Camera average startup time: (\\d+) ms"); + map.put("FirstVideoStartup", "^First Video Startup: (\\d+)"); + map.put("VideoStartup", "^Video average startup time: (\\d+) ms"); + map.put("FirstPreviewTime", "^Avg FirstPreviewTime = (\\d+)"); + } + + /** + * Make sure that parsing works in the expected case + */ + public void testParse() throws Exception { + String output = join( + "First Camera Startup: 1421", /* "FirstCameraStartup" key */ + "Camerastartup time: ", + "Number of loop: 19", + "Individual Camera Startup Time = 1920 ,1929 ,1924 ,1871 ,1840 ,1892 ,1813 " + + ",1927 ,1856 ,1929 ,1911 ,1873 ,1381 ,1888 ,2405 ,1926 ,1840 ,2502 " + + ",2357 ,", + "", + "Camera average startup time: 1946 ms", /* "CameraStartup" key */ + "", + "First Video Startup: 2176", /* "FirstVideoStartup" key */ + "Camera Latency : ", + "Number of loop: 20", + "Avg AutoFocus = 2304", + "Avg mShutterLag = 403", + "Avg mShutterToPictureDisplayedTime = 369", + "Avg mPictureDisplayedToJpegCallbackTime = 50", + "Avg mJpegCallbackFinishTime = 1679", + "Avg FirstPreviewTime = 1340"); + + InputStream iStream = new ByteArrayInputStream(output.getBytes()); + mTestInstance.parseOutputFile(mTestInfo, iStream, null); + assertEquals(mTestInfo, mReportedTestInfo); + assertNotNull(mReportedMetrics); + assertEquals(4, mReportedMetrics.size()); + assertEquals("1946", mReportedMetrics.get("CameraStartup")); + assertEquals("2176", mReportedMetrics.get("FirstVideoStartup")); + assertEquals("1421", mReportedMetrics.get("FirstCameraStartup")); + assertEquals("1340", mReportedMetrics.get("FirstPreviewTime")); + } + } +} diff --git a/src/com/android/media/tests/CameraPerformanceTest.java b/src/com/android/media/tests/CameraPerformanceTest.java new file mode 100644 index 0000000..ebaad68 --- /dev/null +++ b/src/com/android/media/tests/CameraPerformanceTest.java @@ -0,0 +1,672 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.util.FileUtil; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.xmlpull.v1.XmlPullParser; +import org.xmlpull.v1.XmlPullParserException; +import org.xmlpull.v1.XmlPullParserFactory; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This test invocation runs android.hardware.camera2.cts.PerformanceTest - Camera2 API use case + * performance KPIs (Key Performance Indicator), such as camera open time, session creation time, + * shutter lag etc. The KPI data will be parsed and reported. + */ +@OptionClass(alias = "camera-framework") +public class CameraPerformanceTest extends CameraTestBase { + + private static final String TEST_CAMERA_LAUNCH = "testCameraLaunch"; + private static final String TEST_SINGLE_CAPTURE = "testSingleCapture"; + private static final String TEST_REPROCESSING_LATENCY = "testReprocessingLatency"; + private static final String TEST_REPROCESSING_THROUGHPUT = "testReprocessingThroughput"; + + // KPIs to be reported. The key is test methods and the value is KPIs in the method. + private final ImmutableMultimap<String, String> mReportingKpis = + new ImmutableMultimap.Builder<String, String>() + .put(TEST_CAMERA_LAUNCH, "Camera launch time") + .put(TEST_CAMERA_LAUNCH, "Camera start preview time") + .put(TEST_SINGLE_CAPTURE, "Camera capture result latency") + .put(TEST_REPROCESSING_LATENCY, "YUV reprocessing shot to shot latency") + .put(TEST_REPROCESSING_LATENCY, "opaque reprocessing shot to shot latency") + .put(TEST_REPROCESSING_THROUGHPUT, "YUV reprocessing capture latency") + .put(TEST_REPROCESSING_THROUGHPUT, "opaque reprocessing capture latency") + .build(); + + // JSON format keymap, key is test method name and the value is stream name in Json file + private static final ImmutableMap<String, String> METHOD_JSON_KEY_MAP = + new ImmutableMap.Builder<String, String>() + .put(TEST_CAMERA_LAUNCH, "test_camera_launch") + .put(TEST_SINGLE_CAPTURE, "test_single_capture") + .put(TEST_REPROCESSING_LATENCY, "test_reprocessing_latency") + .put(TEST_REPROCESSING_THROUGHPUT, "test_reprocessing_throughput") + .build(); + + private <E extends Number> double getAverage(List<E> list) { + double sum = 0; + int size = list.size(); + for (E num : list) { + sum += num.doubleValue(); + } + if (size == 0) { + return 0.0; + } + return (sum / size); + } + + public CameraPerformanceTest() { + // Set up the default test info. But this is subject to be overwritten by options passed + // from commands. + setTestPackage("android.camera.cts"); + setTestClass("android.hardware.camera2.cts.PerformanceTest"); + setTestRunner("android.support.test.runner.AndroidJUnitRunner"); + setRuKey("camera_framework_performance"); + setTestTimeoutMs(10 * 60 * 1000); // 10 mins + } + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + runInstrumentationTest(listener, new CollectingListener(listener)); + } + + /** + * A listener to collect the output from test run and fatal errors + */ + private class CollectingListener extends DefaultCollectingListener { + + public CollectingListener(ITestInvocationListener listener) { + super(listener); + } + + @Override + public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) { + // Pass the test name for a key in the aggregated metrics, because + // it is used to generate the key of the final metrics to post at the end of test run. + for (Map.Entry<String, String> metric : testMetrics.entrySet()) { + getAggregatedMetrics().put(test.getTestName(), metric.getValue()); + } + } + + @Override + public void handleTestRunEnded( + ITestInvocationListener listener, + long elapsedTime, + Map<String, String> runMetrics) { + // Report metrics at the end of test run. + Map<String, String> result = parseResult(getAggregatedMetrics()); + listener.testRunEnded(getTestDurationMs(), result); + } + } + + /** + * Parse Camera Performance KPIs results and then put them all together to post the final + * report. + * + * @return a {@link HashMap} that contains pairs of kpiName and kpiValue + */ + private Map<String, String> parseResult(Map<String, String> metrics) { + + // if json report exists, return the parse results + CtsJsonResultParser ctsJsonResultParser = new CtsJsonResultParser(); + + if (ctsJsonResultParser.isJsonFileExist()) { + return ctsJsonResultParser.parse(); + } + + Map<String, String> resultsAll = new HashMap<String, String>(); + + CtsResultParserBase parser; + for (Map.Entry<String, String> metric : metrics.entrySet()) { + String testMethod = metric.getKey(); + String testResult = metric.getValue(); + CLog.d("test name %s", testMethod); + CLog.d("test result %s", testResult); + // Probe which result parser should be used. + if (shouldUseCtsXmlResultParser(testResult)) { + parser = new CtsXmlResultParser(); + } else { + parser = new CtsDelimitedResultParser(); + } + + // Get pairs of { KPI name, KPI value } from stdout that each test outputs. + // Assuming that a device has both the front and back cameras, parser will return + // 2 KPIs in HashMap. For an example of testCameraLaunch, + // { + // ("Camera 0 Camera launch time", "379.2"), + // ("Camera 1 Camera launch time", "272.8"), + // } + Map<String, String> testKpis = parser.parse(testResult, testMethod); + for (String k : testKpis.keySet()) { + if (resultsAll.containsKey(k)) { + throw new RuntimeException( + String.format("KPI name (%s) conflicts with the existing names.", k)); + } + } + parser.clear(); + + // Put each result together to post the final result + resultsAll.putAll(testKpis); + } + return resultsAll; + } + + public boolean shouldUseCtsXmlResultParser(String result) { + final String XML_DECLARATION = "<?xml"; + return (result.startsWith(XML_DECLARATION) + || result.startsWith(XML_DECLARATION.toUpperCase())); + } + + /** Data class of CTS test results for Camera framework performance test */ + public static class CtsMetric { + String testMethod; // "testSingleCapture" + String source; // "android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327" + // or "testSingleCapture" (just test method name) + String message; // "Camera 0: Camera capture latency" + String type; // "lower_better" + String unit; // "ms" + String value; // "691.0" (is an average of 736.0 688.0 679.0 667.0 686.0) + String schemaKey; // RU schema key = message (+ testMethodName if needed), derived + + // eg. "android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327" + public static final Pattern SOURCE_REGEX = + Pattern.compile("^(?<package>[a-zA-Z\\d\\._$]+)#(?<method>[a-zA-Z\\d_$]+)(:\\d+)?"); + // eg. "Camera 0: Camera capture latency" + public static final Pattern MESSAGE_REGEX = + Pattern.compile("^Camera\\s+(?<cameraId>\\d+):\\s+(?<kpiName>.*)"); + + CtsMetric( + String testMethod, + String source, + String message, + String type, + String unit, + String value) { + this.testMethod = testMethod; + this.source = source; + this.message = message; + this.type = type; + this.unit = unit; + this.value = value; + this.schemaKey = getRuSchemaKeyName(message); + } + + public boolean matches(String testMethod, String kpiName) { + return (this.testMethod.equals(testMethod) && this.message.endsWith(kpiName)); + } + + public String getRuSchemaKeyName(String message) { + // Note 1: The key shouldn't contain ":" for side by side report. + String schemaKey = message.replace(":", ""); + // Note 2: Two tests testReprocessingLatency & testReprocessingThroughput have the + // same metric names to report results. To make the report key name distinct, + // the test name is added as prefix for these tests for them. + final String[] TEST_NAMES_AS_PREFIX = { + "testReprocessingLatency", "testReprocessingThroughput" + }; + for (String testName : TEST_NAMES_AS_PREFIX) { + if (testMethod.endsWith(testName)) { + schemaKey = String.format("%s_%s", testName, schemaKey); + break; + } + } + return schemaKey; + } + + public String getTestMethodNameInSource(String source) { + Matcher m = SOURCE_REGEX.matcher(source); + if (!m.matches()) { + return source; + } + return m.group("method"); + } + } + + /** + * Base class of CTS test result parser. This is inherited to two derived parsers, + * {@link CtsDelimitedResultParser} for legacy delimiter separated format and + * {@link CtsXmlResultParser} for XML typed format introduced since NYC. + */ + public abstract class CtsResultParserBase { + + protected CtsMetric mSummary; + protected List<CtsMetric> mDetails = new ArrayList<>(); + + /** + * Parse Camera Performance KPIs result first, then leave the only KPIs that matter. + * + * @param result String to be parsed + * @param testMethod test method name used to leave the only metric that matters + * @return a {@link HashMap} that contains kpiName and kpiValue + */ + public abstract Map<String, String> parse(String result, String testMethod); + + protected Map<String, String> filter(List<CtsMetric> metrics, String testMethod) { + Map<String, String> filtered = new HashMap<String, String>(); + for (CtsMetric metric : metrics) { + for (String kpiName : mReportingKpis.get(testMethod)) { + // Post the data only when it matches with the given methods and KPI names. + if (metric.matches(testMethod, kpiName)) { + filtered.put(metric.schemaKey, metric.value); + } + } + } + return filtered; + } + + protected void setSummary(CtsMetric summary) { + mSummary = summary; + } + + protected void addDetail(CtsMetric detail) { + mDetails.add(detail); + } + + protected List<CtsMetric> getDetails() { + return mDetails; + } + + void clear() { + mSummary = null; + mDetails.clear(); + } + } + + /** + * Parses the camera performance test generated by the underlying instrumentation test and + * returns it to test runner for later reporting. + * + * <p>TODO(liuyg): Rename this class to not reference CTS. + * + * <p>Format: (summary message)| |(type)|(unit)|(value) ++++ + * (source)|(message)|(type)|(unit)|(value)... +++ ... + * + * <p>Example: Camera launch average time for Camera 1| |lower_better|ms|586.6++++ + * android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171| Camera 0: Camera open + * time|lower_better|ms|74.0 100.0 70.0 67.0 82.0 +++ + * android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171| Camera 0: Camera configure + * stream time|lower_better|ms|9.0 5.0 5.0 8.0 5.0 ... + * + * <p>See also com.android.cts.util.ReportLog for the format detail. + */ + public class CtsDelimitedResultParser extends CtsResultParserBase { + private static final String LOG_SEPARATOR = "\\+\\+\\+"; + private static final String SUMMARY_SEPARATOR = "\\+\\+\\+\\+"; + private final Pattern mSummaryRegex = + Pattern.compile( + "^(?<message>[^|]+)\\| \\|(?<type>[^|]+)\\|(?<unit>[^|]+)\\|(?<value>[0-9 .]+)"); + private final Pattern mDetailRegex = + Pattern.compile( + "^(?<source>[^|]+)\\|(?<message>[^|]+)\\|(?<type>[^|]+)\\|(?<unit>[^|]+)\\|" + + "(?<values>[0-9 .]+)"); + + @Override + public Map<String, String> parse(String result, String testMethod) { + parseToCtsMetrics(result, testMethod); + parseToCtsMetrics(result, testMethod); + return filter(getDetails(), testMethod); + } + + void parseToCtsMetrics(String result, String testMethod) { + // Split summary and KPIs from stdout passes as parameter. + String[] output = result.split(SUMMARY_SEPARATOR); + if (output.length != 2) { + throw new RuntimeException("Value not in the correct format"); + } + Matcher summaryMatcher = mSummaryRegex.matcher(output[0].trim()); + + // Parse summary. + // Example: "Camera launch average time for Camera 1| |lower_better|ms|586.6++++" + if (summaryMatcher.matches()) { + setSummary( + new CtsMetric( + testMethod, + null, + summaryMatcher.group("message"), + summaryMatcher.group("type"), + summaryMatcher.group("unit"), + summaryMatcher.group("value"))); + } else { + // Fall through since the summary is not posted as results. + CLog.w("Summary not in the correct format"); + } + + // Parse KPIs. + // Example: "android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171|Camera 0: + // Camera open time|lower_better|ms|74.0 100.0 70.0 67.0 82.0 +++" + String[] details = output[1].split(LOG_SEPARATOR); + for (String detail : details) { + Matcher detailMatcher = mDetailRegex.matcher(detail.trim()); + if (detailMatcher.matches()) { + // get average of kpi values + List<Double> values = new ArrayList<>(); + for (String value : detailMatcher.group("values").split("\\s+")) { + values.add(Double.parseDouble(value)); + } + String kpiValue = String.format("%.1f", getAverage(values)); + addDetail( + new CtsMetric( + testMethod, + detailMatcher.group("source"), + detailMatcher.group("message"), + detailMatcher.group("type"), + detailMatcher.group("unit"), + kpiValue)); + } else { + throw new RuntimeException("KPI not in the correct format"); + } + } + } + } + + /** + * Parses the CTS test results in a XML format introduced since NYC. + * Format: + * <Summary> + * <Metric source="android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327" + * message="Camera capture result average latency for all cameras " + * score_type="lower_better" + * score_unit="ms" + * <Value>353.9</Value> + * </Metric> + * </Summary> + * <Detail> + * <Metric source="android.hardware.camera2.cts.PerformanceTest#testSingleCapture:303" + * message="Camera 0: Camera capture latency" + * score_type="lower_better" + * score_unit="ms"> + * <Value>335.0</Value> + * <Value>302.0</Value> + * <Value>316.0</Value> + * </Metric> + * </Detail> + * } + * See also com.android.compatibility.common.util.ReportLog for the format detail. + */ + public class CtsXmlResultParser extends CtsResultParserBase { + private static final String ENCODING = "UTF-8"; + // XML constants + private static final String DETAIL_TAG = "Detail"; + private static final String METRIC_TAG = "Metric"; + private static final String MESSAGE_ATTR = "message"; + private static final String SCORETYPE_ATTR = "score_type"; + private static final String SCOREUNIT_ATTR = "score_unit"; + private static final String SOURCE_ATTR = "source"; + private static final String SUMMARY_TAG = "Summary"; + private static final String VALUE_TAG = "Value"; + private String mTestMethod; + + @Override + public Map<String, String> parse(String result, String testMethod) { + try { + mTestMethod = testMethod; + XmlPullParserFactory factory = XmlPullParserFactory.newInstance(); + XmlPullParser parser = factory.newPullParser(); + parser.setInput(new ByteArrayInputStream(result.getBytes(ENCODING)), ENCODING); + parser.nextTag(); + parse(parser); + return filter(getDetails(), testMethod); + } catch (XmlPullParserException | IOException e) { + throw new RuntimeException("Failed to parse results in XML.", e); + } + } + + /** + * Parses a {@link CtsMetric} from the given XML parser. + * + * @param parser + * @throws IOException + * @throws XmlPullParserException + */ + private void parse(XmlPullParser parser) throws XmlPullParserException, IOException { + parser.require(XmlPullParser.START_TAG, null, SUMMARY_TAG); + parser.nextTag(); + setSummary(parseToCtsMetrics(parser)); + parser.nextTag(); + parser.require(XmlPullParser.END_TAG, null, SUMMARY_TAG); + parser.nextTag(); + if (parser.getName().equals(DETAIL_TAG)) { + while (parser.nextTag() == XmlPullParser.START_TAG) { + addDetail(parseToCtsMetrics(parser)); + } + parser.require(XmlPullParser.END_TAG, null, DETAIL_TAG); + } + } + + CtsMetric parseToCtsMetrics(XmlPullParser parser) + throws IOException, XmlPullParserException { + parser.require(XmlPullParser.START_TAG, null, METRIC_TAG); + String source = parser.getAttributeValue(null, SOURCE_ATTR); + String message = parser.getAttributeValue(null, MESSAGE_ATTR); + String type = parser.getAttributeValue(null, SCORETYPE_ATTR); + String unit = parser.getAttributeValue(null, SCOREUNIT_ATTR); + List<Double> values = new ArrayList<>(); + while (parser.nextTag() == XmlPullParser.START_TAG) { + parser.require(XmlPullParser.START_TAG, null, VALUE_TAG); + values.add(Double.parseDouble(parser.nextText())); + parser.require(XmlPullParser.END_TAG, null, VALUE_TAG); + } + String kpiValue = String.format("%.1f", getAverage(values)); + parser.require(XmlPullParser.END_TAG, null, METRIC_TAG); + return new CtsMetric(mTestMethod, source, message, type, unit, kpiValue); + } + } + + /* + * Parse the Json report from the Json String + * "test_single_capture": + * {"camera_id":"0","camera_capture_latency":[264.0,229.0,229.0,237.0,234.0], + * "camera_capture_result_latency":[230.0,197.0,196.0,204.0,202.0]}," + * "test_reprocessing_latency": + * {"camera_id":"0","format":35,"reprocess_type":"YUV reprocessing", + * "capture_message":"shot to shot latency","latency":[102.0,101.0,99.0,99.0,100.0,101.0], + * "camera_reprocessing_shot_to_shot_average_latency":100.33333333333333}, + * + * TODO: move this to a seperate class + */ + public class CtsJsonResultParser { + + // report json file set in + // cts/tools/cts-tradefed/res/config/cts-preconditions.xml + private static final String JSON_RESULT_FILE = + "/sdcard/report-log-files/CtsCameraTestCases.reportlog.json"; + private static final String CAMERA_ID_KEY = "camera_id"; + private static final String AVERAGE_LATENCY_KEY = "average_latency"; + private static final String REPROCESS_TYPE_KEY = "reprocess_type"; + private static final String CAPTURE_MESSAGE_KEY = "capture_message"; + private static final String LATENCY_KEY = "latency"; + + public Map<String, String> parse() { + + Map<String, String> metrics = new HashMap<>(); + + String jsonString = getFormatedJsonReportFromFile(); + if (null == jsonString) { + throw new RuntimeException("Get null json report string."); + } + + Map<String, List<Double>> metricsData = new HashMap<>(); + + try { + JSONObject jsonObject = new JSONObject(jsonString); + + for (String testMethod : METHOD_JSON_KEY_MAP.keySet()) { + + JSONArray jsonArray = + (JSONArray) jsonObject.get(METHOD_JSON_KEY_MAP.get(testMethod)); + + switch (testMethod) { + case TEST_REPROCESSING_THROUGHPUT: + case TEST_REPROCESSING_LATENCY: + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject element = jsonArray.getJSONObject(i); + + // create a kpiKey from camera id, + // reprocess type and capture message + String cameraId = element.getString(CAMERA_ID_KEY); + String reprocessType = element.getString(REPROCESS_TYPE_KEY); + String captureMessage = element.getString(CAPTURE_MESSAGE_KEY); + String kpiKey = + String.format( + "%s_Camera %s %s %s", + testMethod, + cameraId, + reprocessType, + captureMessage); + + // read the data array from json object + JSONArray jsonDataArray = element.getJSONArray(LATENCY_KEY); + if (!metricsData.containsKey(kpiKey)) { + List<Double> list = new ArrayList<>(); + metricsData.put(kpiKey, list); + } + for (int j = 0; j < jsonDataArray.length(); j++) { + metricsData.get(kpiKey).add(jsonDataArray.getDouble(j)); + } + } + break; + case TEST_SINGLE_CAPTURE: + case TEST_CAMERA_LAUNCH: + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject element = jsonArray.getJSONObject(i); + + String cameraid = element.getString(CAMERA_ID_KEY); + for (String kpiName : mReportingKpis.get(testMethod)) { + + // the json key is all lower case + String jsonKey = kpiName.toLowerCase().replace(" ", "_"); + String kpiKey = + String.format("Camera %s %s", cameraid, kpiName); + if (!metricsData.containsKey(kpiKey)) { + List<Double> list = new ArrayList<>(); + metricsData.put(kpiKey, list); + } + JSONArray jsonDataArray = element.getJSONArray(jsonKey); + for (int j = 0; j < jsonDataArray.length(); j++) { + metricsData.get(kpiKey).add(jsonDataArray.getDouble(j)); + } + } + } + break; + default: + break; + } + } + } catch (JSONException e) { + CLog.w("JSONException: %s in string %s", e.getMessage(), jsonString); + } + + // take the average of all data for reporting + for (String kpiKey : metricsData.keySet()) { + String kpiValue = String.format("%.1f", getAverage(metricsData.get(kpiKey))); + metrics.put(kpiKey, kpiValue); + } + return metrics; + } + + public boolean isJsonFileExist() { + try { + return getDevice().doesFileExist(JSON_RESULT_FILE); + } catch (DeviceNotAvailableException e) { + throw new RuntimeException("Failed to check json report file on device.", e); + } + } + + /* + * read json report file on the device + */ + private String getFormatedJsonReportFromFile() { + String jsonString = null; + try { + // pull the json report file from device + File outputFile = FileUtil.createTempFile("json", ".txt"); + getDevice().pullFile(JSON_RESULT_FILE, outputFile); + jsonString = reformatJsonString(FileUtil.readStringFromFile(outputFile)); + } catch (IOException e) { + CLog.w("Couldn't parse the output json log file: ", e); + } catch (DeviceNotAvailableException e) { + CLog.w("Could not pull file: %s, error: %s", JSON_RESULT_FILE, e); + } + return jsonString; + } + + // Reformat the json file to remove duplicate keys + private String reformatJsonString(String jsonString) { + + final String TEST_METRICS_PATTERN = "\\\"([a-z0-9_]*)\\\":(\\{[^{}]*\\})"; + StringBuilder newJsonBuilder = new StringBuilder(); + // Create map of stream names and json objects. + HashMap<String, List<String>> jsonMap = new HashMap<>(); + Pattern p = Pattern.compile(TEST_METRICS_PATTERN); + Matcher m = p.matcher(jsonString); + while (m.find()) { + String key = m.group(1); + String value = m.group(2); + if (!jsonMap.containsKey(key)) { + jsonMap.put(key, new ArrayList<String>()); + } + jsonMap.get(key).add(value); + } + // Rewrite json string as arrays. + newJsonBuilder.append("{"); + boolean firstLine = true; + for (String key : jsonMap.keySet()) { + if (!firstLine) { + newJsonBuilder.append(","); + } else { + firstLine = false; + } + newJsonBuilder.append("\"").append(key).append("\":["); + boolean firstValue = true; + for (String stream : jsonMap.get(key)) { + if (!firstValue) { + newJsonBuilder.append(","); + } else { + firstValue = false; + } + newJsonBuilder.append(stream); + } + newJsonBuilder.append("]"); + } + newJsonBuilder.append("}"); + return newJsonBuilder.toString(); + } + } +} diff --git a/src/com/android/media/tests/CameraSettingsTest.java b/src/com/android/media/tests/CameraSettingsTest.java new file mode 100644 index 0000000..d5fe0dd --- /dev/null +++ b/src/com/android/media/tests/CameraSettingsTest.java @@ -0,0 +1,222 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.IDevice; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.android.tradefed.config.Option; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.BugreportCollector; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.StreamUtil; + +import org.junit.Assert; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.ListIterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Camera zoom stress test that increments the camera's zoom level across the + * entire range [min, max], taking a picture at each level. + */ +public class CameraSettingsTest implements IDeviceTest, IRemoteTest { + + private static final String ZOOM_STANZA = "testStressCameraZoom"; + private static final String SCENE_MODES_STANZA = "testStressCameraSceneModes"; + private static final Pattern EXPECTED_LOOP_COUNT_PATTERN = + Pattern.compile("(Total number of loops:)(\\s*)(\\d+)"); + private static final Pattern ACTUAL_LOOP_COUNT_PATTERN = + Pattern.compile("(No of loop:)(.*,\\s)(\\d+)$"); + + private static final String TEST_CLASS_NAME = + "com.android.mediaframeworktest.stress.CameraStressTest"; + private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest"; + private static final String TEST_RUNNER_NAME = + "com.android.mediaframeworktest.CameraStressTestRunner"; + private static final String TEST_RU = "CameraApplicationStress"; + + private final String mOutputPath = "cameraStressOutput.txt"; + private static final int MAX_TIME_OUT = 90 * 60 * 1000; //90 mins + + @Option(name="testMethodName", description="Used to specify a specific test method to run") + private String mTestMethodName = null; + + ITestDevice mTestDevice = null; + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + Assert.assertNotNull(mTestDevice); + + IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, + TEST_RUNNER_NAME, mTestDevice.getIDevice()); + runner.setClassName(TEST_CLASS_NAME); + + if (mTestMethodName != null) { + runner.setMethodName(TEST_CLASS_NAME, mTestMethodName); + } + runner.setMaxTimeToOutputResponse(MAX_TIME_OUT, TimeUnit.MILLISECONDS); + + BugreportCollector bugListener = new BugreportCollector(listener, mTestDevice); + bugListener.addPredicate(BugreportCollector.AFTER_FAILED_TESTCASES); + bugListener.setDescriptiveName(this.getClass().getName()); + Assert.assertTrue(mTestDevice.runInstrumentationTests(runner, bugListener)); + + Map<String, String> metrics = parseOutputFile(); + reportMetrics(bugListener, TEST_RU, metrics); + cleanupDevice(); + } + + /** + * {@inheritDoc} + */ + @Override + public void setDevice(ITestDevice device) { + mTestDevice = device; + } + + /** + * {@inheritDoc} + */ + @Override + public ITestDevice getDevice() { + return mTestDevice; + } + + /** + * Wipes the device's external memory of test collateral from prior runs. + * + * @throws DeviceNotAvailableException If the device is unavailable or + * something happened while deleting files + */ + private void cleanupDevice() throws DeviceNotAvailableException { + String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath)); + } + + /** + * Parses the output file generated by the underlying instrumentation test + * and returns it to the main driver for later reporting. + * + * @return The {@link Map} that contains metrics for the test. + * @throws DeviceNotAvailableException If the device is unavailable or + * something happened while deleting files + */ + private Map<String, String> parseOutputFile() throws DeviceNotAvailableException { + File outputFile = null; + BufferedReader reader = null; + ArrayList<String> lines = new ArrayList<String>(); + String line = null; + String key = null; + Integer expectedCount = null; + Integer actualCount = null; + ListIterator<String> listIterator = null; + Map<String, String> metrics = new HashMap<String, String>(); + + // Read in data + try { + outputFile = mTestDevice.pullFileFromExternal(mOutputPath); + reader = new BufferedReader(new FileReader(outputFile)); + + while ((line = reader.readLine()) != null) { + if (!line.isEmpty()) { + lines.add(line); + } + } + } catch (IOException e) { + CLog.e(String.format("IOException reading from file: %s", e.toString())); + } finally { + StreamUtil.close(reader); + } + + // Output file looks like: + // Test name: + // Total number of loops: 123 + // No of loop: 0, 1, 2, 3, ..., 122 (0 based) + // Note that the actual count should be +1 as the # of loop is 0 based. + listIterator = lines.listIterator(); + + while (listIterator.hasNext()) { + line = listIterator.next(); + CLog.d(String.format("Parsing line: \"%s\"", line)); + + if (ZOOM_STANZA.equals(line)) { + key = "CameraZoom"; + } else if (SCENE_MODES_STANZA.equals(line)) { + key = "CameraSceneMode"; + } + + Matcher expectedMatcher = EXPECTED_LOOP_COUNT_PATTERN.matcher(line); + if (expectedMatcher.matches()) { + expectedCount = Integer.valueOf(expectedMatcher.group(3)); + CLog.d(String.format("Found expected count for key \"%s\": %s", + key, expectedCount)); + } + + Matcher actualMatcher = ACTUAL_LOOP_COUNT_PATTERN.matcher(line); + if (actualMatcher.matches()) { + actualCount = 1 + Integer.valueOf(actualMatcher.group(3)); + CLog.d(String.format("Found actual count for key \"%s\": %s", key, actualCount)); + } + + if ((key != null) && (expectedCount != null) && (actualCount != null)) { + metrics.put(key, String.format("%d", actualCount)); + key = null; + expectedCount = null; + actualCount = null; + } + } + + return metrics; + } + + /** + * Report run metrics by creating an empty test run to stick them in. + * + * @param listener The {@link ITestInvocationListener} of test results + * @param runName The test name + * @param metrics The {@link Map} that contains metrics for the given test + */ + private void reportMetrics(ITestInvocationListener listener, String runName, + Map<String, String> metrics) { + InputStreamSource bugreport = mTestDevice.getBugreport(); + listener.testLog("bugreport", LogDataType.BUGREPORT, bugreport); + bugreport.cancel(); + + CLog.d(String.format("About to report metrics: %s", metrics)); + listener.testRunStarted(runName, 0); + listener.testRunEnded(0, metrics); + } +} diff --git a/src/com/android/media/tests/CameraShotLatencyTest.java b/src/com/android/media/tests/CameraShotLatencyTest.java new file mode 100644 index 0000000..f72ded7 --- /dev/null +++ b/src/com/android/media/tests/CameraShotLatencyTest.java @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Camera shot latency test + * + * Runs Camera device performance test to measure time from taking a shot to saving a file on disk + * and time from shutter to shutter in fully saturated case. + */ +@OptionClass(alias = "camera-shot-latency") +public class CameraShotLatencyTest extends CameraTestBase { + + private static final Pattern STATS_REGEX = Pattern.compile( + "^(?<average>[0-9.]+)\\|(?<values>[0-9 .-]+)"); + + public CameraShotLatencyTest() { + setTestPackage("com.google.android.camera"); + setTestClass("com.android.camera.latency.CameraShotLatencyTest"); + setTestRunner("android.test.InstrumentationTestRunner"); + setRuKey("CameraShotLatency"); + setTestTimeoutMs(60 * 60 * 1000); // 1 hour + } + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + runInstrumentationTest(listener, new CollectingListener(listener)); + } + + /** + * A listener to collect the output from test run and fatal errors + */ + private class CollectingListener extends DefaultCollectingListener { + + public CollectingListener(ITestInvocationListener listener) { + super(listener); + } + + @Override + public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) { + // Test metrics accumulated will be posted at the end of test run. + getAggregatedMetrics().putAll(parseResults(test.getTestName(), testMetrics)); + } + + public Map<String, String> parseResults(String testName, Map<String, String> testMetrics) { + // Parse shot latency stats from the instrumentation result. + // Format : <metric_key>=<average_of_latency>|<raw_data> + // Example: + // ElapsedTimeMs=1805|1725 1747 ... 2078 + Map<String, String> parsed = new HashMap<String, String>(); + for (Map.Entry<String, String> metric : testMetrics.entrySet()) { + Matcher matcher = STATS_REGEX.matcher(metric.getValue()); + if (matcher.matches()) { + // Key name consists of a pair of test name and metric name. + String keyName = String.format("%s_%s", testName, metric.getKey()); + parsed.put(keyName, matcher.group("average")); + } else { + CLog.w(String.format("Stats not in correct format: %s", metric.getValue())); + } + } + return parsed; + } + } +} diff --git a/src/com/android/media/tests/CameraShotToShotLatencyTest.java b/src/com/android/media/tests/CameraShotToShotLatencyTest.java new file mode 100644 index 0000000..c30d5c0 --- /dev/null +++ b/src/com/android/media/tests/CameraShotToShotLatencyTest.java @@ -0,0 +1,184 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.IDevice; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.BugreportCollector; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; + +import org.junit.Assert; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class CameraShotToShotLatencyTest implements IDeviceTest, IRemoteTest { + + private static final Pattern MEAN_PATTERN = + Pattern.compile("(Shot to shot latency - mean:)(\\s*)(\\d+\\.\\d*)"); + private static final Pattern STANDARD_DEVIATION_PATTERN = + Pattern.compile("(Shot to shot latency - standard deviation:)(\\s*)(\\d+\\.\\d*)"); + + private static final String TEST_CLASS_NAME = "com.android.camera.stress.ShotToShotLatency"; + private static final String TEST_PACKAGE_NAME = "com.google.android.camera.tests"; + private static final String TEST_RUNNER_NAME = "android.test.InstrumentationTestRunner"; + + private static final String LATENCY_KEY_MEAN = "Shot2ShotLatencyMean"; + private static final String LATENCY_KEY_SD = "Shot2ShotLatencySD"; + private static final String TEST_RU = "CameraLatency"; + + private final String mOutputPath = "mediaStressOut.txt"; + ITestDevice mTestDevice = null; + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + Assert.assertNotNull(mTestDevice); + + IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, + TEST_RUNNER_NAME, mTestDevice.getIDevice()); + runner.setClassName(TEST_CLASS_NAME); + + BugreportCollector bugListener = new BugreportCollector(listener, mTestDevice); + bugListener.addPredicate(BugreportCollector.AFTER_FAILED_TESTCASES); + bugListener.setDescriptiveName(this.getClass().getName()); + Assert.assertTrue(mTestDevice.runInstrumentationTests(runner, bugListener)); + + Map<String, String> metrics = parseOutputFile(); + reportMetrics(bugListener, TEST_RU, metrics); + cleanupDevice(); + } + + /** + * {@inheritDoc} + */ + @Override + public void setDevice(ITestDevice device) { + mTestDevice = device; + } + + /** + * {@inheritDoc} + */ + @Override + public ITestDevice getDevice() { + return mTestDevice; + } + + /** + * Wipes the device's external memory of test collateral from prior runs. + * Note that all photos on the test device will be removed. + * @throws DeviceNotAvailableException If the device is unavailable or + * something happened while deleting files + */ + private void cleanupDevice() throws DeviceNotAvailableException { + String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + mTestDevice.executeShellCommand(String.format("rm -r %s/DCIM", extStore)); + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath)); + } + + /** + * Parses the output file generated by the underlying instrumentation test + * and returns the metrics to the main driver for later reporting. + * @return The {@link Map} that contains metrics for the test. + * @throws DeviceNotAvailableException If the device is unavailable or + * something happened while deleting files + */ + private Map<String, String> parseOutputFile() throws DeviceNotAvailableException { + BufferedReader reader = null; + File outputFile = null; + String lineMean = null, lineSd = null; + Matcher m = null; + Map<String, String> metrics = new HashMap<String, String>(); + + // Read in data + // Output file is only 2 lines and should look something like: + // "Shot to shot latency - mean: 1234.5678901" + // "Shot to shot latency - standard deviation: 123.45678901" + try { + outputFile = mTestDevice.pullFileFromExternal(mOutputPath); + reader = new BufferedReader(new FileReader(outputFile)); + + lineMean = reader.readLine(); + lineSd = reader.readLine(); + + if ((lineMean == null) || (lineSd == null)) { + CLog.e(String.format("Unable to find output data; hit EOF: \nmean:%s\nsd:%s", + lineMean, lineSd)); + } else { + m = MEAN_PATTERN.matcher(lineMean); + if (m.matches()) { + metrics.put(LATENCY_KEY_MEAN, m.group(3)); + } else { + CLog.e(String.format("Unable to find mean: %s", lineMean)); + } + + m = STANDARD_DEVIATION_PATTERN.matcher(lineSd); + if (m.matches()) { + metrics.put(LATENCY_KEY_SD, m.group(3)); + } else { + CLog.e(String.format("Unable to find standard deviation: %s", lineSd)); + } + } + } catch (IOException e) { + CLog.e(String.format("IOException reading from file: %s", e.toString())); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException e) { + CLog.e(String.format("IOException closing file: %s", e.toString())); + } + } + } + + return metrics; + } + + /** + * Report run metrics by creating an empty test run to stick them in. + * @param listener The {@link ITestInvocationListener} of test results + * @param runName The test name + * @param metrics The {@link Map} that contains metrics for the given test + */ + private void reportMetrics(ITestInvocationListener listener, String runName, + Map<String, String> metrics) { + InputStreamSource bugreport = mTestDevice.getBugreport(); + listener.testLog("bugreport", LogDataType.BUGREPORT, bugreport); + bugreport.cancel(); + + CLog.d(String.format("About to report metrics: %s", metrics)); + listener.testRunStarted(runName, 0); + listener.testRunEnded(0, metrics); + } +} diff --git a/src/com/android/media/tests/CameraStartupTest.java b/src/com/android/media/tests/CameraStartupTest.java new file mode 100644 index 0000000..c19cd2f --- /dev/null +++ b/src/com/android/media/tests/CameraStartupTest.java @@ -0,0 +1,236 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.Option; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.targetprep.BuildError; +import com.android.tradefed.targetprep.ITargetPreparer; +import com.android.tradefed.targetprep.TargetSetupError; +import com.android.tradefed.targetprep.TemperatureThrottlingWaiter; +import com.android.tradefed.util.MultiMap; + +import org.junit.Assert; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Camera app startup test + * + * Runs CameraActivityTest to measure Camera startup time and reports the metrics. + */ +@OptionClass(alias = "camera-startup") +public class CameraStartupTest extends CameraTestBase { + + private static final Pattern STATS_REGEX = Pattern.compile( + "^(?<coldStartup>[0-9.]+)\\|(?<warmStartup>[0-9.]+)\\|(?<values>[0-9 .-]+)"); + private static final String PREFIX_COLD_STARTUP = "Cold"; + // all metrics are expected to be less than 10 mins and greater than 0. + private static final int METRICS_MAX_THRESHOLD_MS = 10 * 60 * 1000; + private static final int METRICS_MIN_THRESHOLD_MS = 0; + private static final String INVALID_VALUE = "-1"; + + @Option(name="num-test-runs", description="The number of test runs. A instrumentation " + + "test will be repeatedly executed. Then it posts the average of test results.") + private int mNumTestRuns = 1; + + @Option(name="delay-between-test-runs", description="Time delay between multiple test runs, " + + "in msecs. Used to wait for device to cool down. " + + "Note that this will be ignored when TemperatureThrottlingWaiter is configured.") + private long mDelayBetweenTestRunsMs = 120 * 1000; // 2 minutes + + private MultiMap<String, String> mMultipleRunMetrics = new MultiMap<String, String>(); + private Map<String, String> mAverageMultipleRunMetrics = new HashMap<String, String>(); + private long mTestRunsDurationMs = 0; + + public CameraStartupTest() { + setTestPackage("com.google.android.camera"); + setTestClass("com.android.camera.latency.CameraStartupTest"); + setTestRunner("android.test.InstrumentationTestRunner"); + setRuKey("CameraAppStartup"); + setTestTimeoutMs(60 * 60 * 1000); // 1 hour + } + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + runMultipleInstrumentationTests(listener, mNumTestRuns); + } + + private void runMultipleInstrumentationTests(ITestInvocationListener listener, int numTestRuns) + throws DeviceNotAvailableException { + Assert.assertTrue(numTestRuns > 0); + + mTestRunsDurationMs = 0; + for (int i = 0; i < numTestRuns; ++i) { + CLog.v("Running multiple instrumentation tests... [%d/%d]", i + 1, numTestRuns); + CollectingListener singleRunListener = new CollectingListener(listener); + runInstrumentationTest(listener, singleRunListener); + mTestRunsDurationMs += getTestDurationMs(); + + if (singleRunListener.hasFailedTests() || + singleRunListener.hasTestRunFatalError()) { + exitTestRunsOnError(listener, singleRunListener.getErrorMessage()); + return; + } + if (i + 1 < numTestRuns) { // Skipping preparation on the last run + postSetupTestRun(); + } + } + + // Post the average of metrics collected in multiple instrumentation test runs. + postMultipleRunMetrics(listener); + CLog.v("multiple instrumentation tests end"); + } + + private void exitTestRunsOnError(ITestInvocationListener listener, String errorMessage) { + CLog.e("The instrumentation result not found. Test runs may have failed due to exceptions." + + " Test results will not be posted. errorMsg: %s", errorMessage); + listener.testRunFailed(errorMessage); + listener.testRunEnded(mTestRunsDurationMs, Collections.<String, String>emptyMap()); + } + + private void postMultipleRunMetrics(ITestInvocationListener listener) { + listener.testRunEnded(mTestRunsDurationMs, getAverageMultipleRunMetrics()); + } + + private void postSetupTestRun() throws DeviceNotAvailableException { + // Reboot for a cold start up of Camera application + CLog.d("Cold start: Rebooting..."); + getDevice().reboot(); + + // Wait for device to cool down to target temperature + // Use TemperatureThrottlingWaiter if configured, otherwise just wait for + // a specific amount of time. + CLog.d("Cold start: Waiting for device to cool down..."); + boolean usedTemperatureThrottlingWaiter = false; + for (ITargetPreparer preparer : mConfiguration.getTargetPreparers()) { + if (preparer instanceof TemperatureThrottlingWaiter) { + usedTemperatureThrottlingWaiter = true; + try { + preparer.setUp(getDevice(), null); + } catch (TargetSetupError e) { + CLog.w("No-op even when temperature is still high after wait timeout. " + + "error: %s", e.getMessage()); + } catch (BuildError e) { + // This should not happen. + } + } + } + if (!usedTemperatureThrottlingWaiter) { + getRunUtil().sleep(mDelayBetweenTestRunsMs); + } + CLog.d("Device gets prepared for the next test run."); + } + + // Call this function once at the end to get the average. + private Map<String, String> getAverageMultipleRunMetrics() { + Assert.assertTrue(mMultipleRunMetrics.size() > 0); + + Set<String> keys = mMultipleRunMetrics.keySet(); + mAverageMultipleRunMetrics.clear(); + for (String key : keys) { + int sum = 0; + int size = 0; + boolean isInvalid = false; + for (String valueString : mMultipleRunMetrics.get(key)) { + int value = Integer.parseInt(valueString); + // If value is out of valid range, skip posting the result associated with the key + if (value > METRICS_MAX_THRESHOLD_MS || value < METRICS_MIN_THRESHOLD_MS) { + isInvalid = true; + break; + } + sum += value; + ++size; + } + + String valueString = INVALID_VALUE; + if (isInvalid) { + CLog.w("Value is out of valid range. Key: %s ", key); + } else { + valueString = String.format("%d", (sum / size)); + } + mAverageMultipleRunMetrics.put(key, valueString); + } + return mAverageMultipleRunMetrics; + } + + /** + * A listener to collect the output from test run and fatal errors + */ + private class CollectingListener extends DefaultCollectingListener { + + public CollectingListener(ITestInvocationListener listener) { + super(listener); + } + + @Override + public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) { + // Test metrics accumulated will be posted at the end of test run. + getAggregatedMetrics().putAll(parseResults(testMetrics)); + } + + @Override + public void handleTestRunEnded(ITestInvocationListener listener, long elapsedTime, + Map<String, String> runMetrics) { + // Do not post aggregated metrics from a single run to a dashboard. Instead, it needs + // to collect all metrics from multiple test runs. + mMultipleRunMetrics.putAll(getAggregatedMetrics()); + } + + public Map<String, String> parseResults(Map<String, String> testMetrics) { + // Parse activity time stats from the instrumentation result. + // Format : <metric_key>=<cold_startup>|<average_of_warm_startups>|<all_startups> + // Example: + // VideoStartupTimeMs=1098|1184.6|1098 1222 ... 788 + // VideoOnCreateTimeMs=138|103.3|138 114 ... 114 + // VideoOnResumeTimeMs=39|40.4|39 36 ... 41 + // VideoFirstPreviewFrameTimeMs=0|0.0|0 0 ... 0 + // CameraStartupTimeMs=2388|1045.4|2388 1109 ... 746 + // CameraOnCreateTimeMs=574|122.7|574 124 ... 109 + // CameraOnResumeTimeMs=610|504.6|610 543 ... 278 + // CameraFirstPreviewFrameTimeMs=0|0.0|0 0 ... 0 + // + // Then report only the first two startup time of cold startup and average warm startup. + Map<String, String> parsed = new HashMap<String, String>(); + for (Map.Entry<String, String> metric : testMetrics.entrySet()) { + Matcher matcher = STATS_REGEX.matcher(metric.getValue()); + String keyName = metric.getKey(); + String coldStartupValue = INVALID_VALUE; + String warmStartupValue = INVALID_VALUE; + if (matcher.matches()) { + coldStartupValue = matcher.group("coldStartup"); + warmStartupValue = matcher.group("warmStartup"); + } + parsed.put(PREFIX_COLD_STARTUP + keyName, coldStartupValue); + parsed.put(keyName, warmStartupValue); + } + return parsed; + } + } +} diff --git a/src/com/android/media/tests/CameraStressTest.java b/src/com/android/media/tests/CameraStressTest.java new file mode 100644 index 0000000..def3f39 --- /dev/null +++ b/src/com/android/media/tests/CameraStressTest.java @@ -0,0 +1,513 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.IDevice; +import com.android.ddmlib.Log; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.android.tradefed.config.Option; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.result.CollectingTestListener; +import com.android.tradefed.result.FileInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.RegexTrie; +import com.android.tradefed.util.StreamUtil; + +import junit.framework.TestCase; + +import org.junit.Assert; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Runs the Camera stress testcases. + * FIXME: more details + * <p/> + * Note that this test will not run properly unless /sdcard is mounted and writable. + */ +public class CameraStressTest implements IDeviceTest, IRemoteTest { + private static final String LOG_TAG = "CameraStressTest"; + + ITestDevice mTestDevice = null; + + // Constants for running the tests + private static final String TEST_PACKAGE_NAME = "com.google.android.camera.tests"; + private static final String TEST_RUNNER = "com.android.camera.stress.CameraStressTestRunner"; + + //Max test timeout - 3 hrs + private static final int MAX_TEST_TIMEOUT = 3 * 60 * 60 * 1000; + + private final String mOutputPath = "mediaStressOut.txt"; + + /** + * Stores the test cases that we should consider running. + * + * <p>This currently consists of "startup" and "latency" + */ + private List<TestInfo> mTestCases = new ArrayList<>(); + + // Options for the running the gCam test + @Option(name = "gCam", description = "Run gCam back image capture test") + private boolean mGcam = false; + + /** + * A struct that contains useful info about the tests to run + */ + static class TestInfo { + public String mTestName = null; + public String mClassName = null; + public String mTestMetricsName = null; + public Map<String, String> mInstrumentationArgs = new HashMap<>(); + public RegexTrie<String> mPatternMap = new RegexTrie<>(); + + @Override + public String toString() { + return String.format("TestInfo: name(%s) class(%s) metric(%s) patterns(%s)", mTestName, + mClassName, mTestMetricsName, mPatternMap); + } + } + + /** + * Set up the pattern map for parsing output files + * <p/> + * Exposed for unit meta-testing + */ + static RegexTrie<String> getPatternMap() { + RegexTrie<String> patMap = new RegexTrie<>(); + patMap.put("SwitchPreview", "^Camera Switch Mode:"); + + // For versions of the on-device test that don't differentiate between front and back camera + patMap.put("ImageCapture", "^Camera Image Capture"); + patMap.put("VideoRecording", "^Camera Video Capture"); + + // For versions that do differentiate + patMap.put("FrontImageCapture", "^Front Camera Image Capture"); + patMap.put("ImageCapture", "^Back Camera Image Capture"); + patMap.put("FrontVideoRecording", "^Front Camera Video Capture"); + patMap.put("VideoRecording", "^Back Camera Video Capture"); + + // Actual metrics to collect for a given key + patMap.put("loopCount", "^No of loops :(\\d+)"); + patMap.put("iters", "^loop:.+,(\\d+)"); + + return patMap; + } + + /** + * Set up the configurations for the test cases we want to run + */ + private void testInfoSetup() { + RegexTrie<String> patMap = getPatternMap(); + TestInfo t = new TestInfo(); + + if (mGcam) { + // Back Image capture stress test for gCam + t.mTestName = "testBackImageCapture"; + t.mClassName = "com.android.camera.stress.ImageCapture"; + t.mTestMetricsName = "GCamApplicationStress"; + t.mInstrumentationArgs.put("image_iterations", Integer.toString(100)); + t.mPatternMap = patMap; + mTestCases.add(t); + + } else { + // Image capture stress test + t.mTestName = "imagecap"; + t.mClassName = "com.android.camera.stress.ImageCapture"; + t.mTestMetricsName = "CameraApplicationStress"; + t.mInstrumentationArgs.put("image_iterations", Integer.toString(100)); + t.mPatternMap = patMap; + mTestCases.add(t); + + // Image capture stress test + t = new TestInfo(); + t.mTestName = "videocap"; + t.mClassName = "com.android.camera.stress.VideoCapture"; + t.mTestMetricsName = "CameraApplicationStress"; + t.mInstrumentationArgs.put("video_iterations", Integer.toString(100)); + t.mPatternMap = patMap; + mTestCases.add(t); + + // "SwitchPreview" stress test + t = new TestInfo(); + t.mTestName = "switch"; + t.mClassName = "com.android.camera.stress.SwitchPreview"; + t.mTestMetricsName = "CameraApplicationStress"; + t.mPatternMap = patMap; + mTestCases.add(t); + } + } + + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + Assert.assertNotNull(mTestDevice); + testInfoSetup(); + for (TestInfo test : mTestCases) { + cleanTmpFiles(); + executeTest(test, listener); + logOutputFiles(test, listener); + } + + cleanTmpFiles(); + } + + private void executeTest(TestInfo test, ITestInvocationListener listener) + throws DeviceNotAvailableException { + IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, + TEST_RUNNER, mTestDevice.getIDevice()); + CollectingTestListener auxListener = new CollectingTestListener(); + + runner.setClassName(test.mClassName); + runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS); + if (mGcam){ + runner.setMethodName(test.mClassName, test.mTestName); + } + + Set<String> argumentKeys = test.mInstrumentationArgs.keySet(); + for (String s : argumentKeys) { + runner.addInstrumentationArg(s, test.mInstrumentationArgs.get(s)); + } + + mTestDevice.runInstrumentationTests(runner, listener, auxListener); + + // Grab a bugreport if warranted + if (auxListener.hasFailedTests()) { + Log.e(LOG_TAG, String.format("Grabbing bugreport after test '%s' finished with " + + "%d failures.", test.mTestName, auxListener.getNumAllFailedTests())); + InputStreamSource bugreport = mTestDevice.getBugreport(); + listener.testLog(String.format("bugreport-%s.txt", test.mTestName), + LogDataType.BUGREPORT, bugreport); + bugreport.cancel(); + } + } + + /** + * Clean up temp files from test runs + * <p /> + * Note that all photos on the test device will be removed + */ + private void cleanTmpFiles() throws DeviceNotAvailableException { + String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + mTestDevice.executeShellCommand(String.format("rm -r %s/DCIM/Camera", extStore)); + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath)); + } + + /** + * Pull the output file from the device, add it to the logs, and also parse out the relevant + * test metrics and report them. Additionally, pull the memory file (if it exists) and report + * it. + */ + private void logOutputFiles(TestInfo test, ITestInvocationListener listener) + throws DeviceNotAvailableException { + File outputFile = null; + InputStreamSource outputSource = null; + try { + outputFile = mTestDevice.pullFileFromExternal(mOutputPath); + + if (outputFile == null) { + return; + } + + // Upload a verbatim copy of the output file + Log.d(LOG_TAG, String.format("Sending %d byte file %s into the logosphere!", + outputFile.length(), outputFile)); + outputSource = new FileInputStreamSource(outputFile); + listener.testLog(String.format("output-%s.txt", test.mTestName), LogDataType.TEXT, + outputSource); + + // Parse the output file to upload aggregated metrics + parseOutputFile(test, new FileInputStream(outputFile), listener); + } catch (IOException e) { + Log.e(LOG_TAG, String.format("IOException while reading or parsing output file: %s", e)); + } finally { + FileUtil.deleteFile(outputFile); + StreamUtil.cancel(outputSource); + } + } + + /** + * Parse the relevant metrics from the Instrumentation test output file + */ + private void parseOutputFile(TestInfo test, InputStream dataStream, + ITestInvocationListener listener) { + Map<String, String> runMetrics = new HashMap<>(); + + String contents; + try { + contents = StreamUtil.getStringFromStream(dataStream); + } catch (IOException e) { + Log.e(LOG_TAG, String.format("Got IOException during %s test processing: %s", + test.mTestName, e)); + return; + } + + String key = null; + Integer countExpected = null; + Integer countActual = null; + + List<String> lines = Arrays.asList(contents.split("\n")); + ListIterator<String> lineIter = lines.listIterator(); + String line; + while (lineIter.hasNext()) { + line = lineIter.next(); + List<List<String>> capture = new ArrayList<>(1); + String pattern = test.mPatternMap.retrieve(capture, line); + if (pattern != null) { + if ("loopCount".equals(pattern)) { + // First capture in first (only) string + countExpected = Integer.parseInt(capture.get(0).get(0)); + } else if ("iters".equals(pattern)) { + // First capture in first (only) string + countActual = Integer.parseInt(capture.get(0).get(0)); + + if (countActual != null) { + // countActual starts counting at 0 + countActual += 1; + } + } else { + // Assume that the pattern is the name of a key + + // commit, if there was a previous key + if (key != null) { + int value = coalesceLoopCounts(countActual, countExpected); + runMetrics.put(key, Integer.toString(value)); + } + + key = pattern; + countExpected = null; + countActual = null; + } + + Log.d(LOG_TAG, String.format("Got %s key '%s' and captures '%s'", + test.mTestName, key, capture.toString())); + } else if (line.isEmpty()) { + // ignore + continue; + } else { + Log.e(LOG_TAG, String.format("Got unmatched line: %s", line)); + continue; + } + + // commit the final key, if there was one + if (key != null) { + int value = coalesceLoopCounts(countActual, countExpected); + runMetrics.put(key, Integer.toString(value)); + } + } + + reportMetrics(listener, test, runMetrics); + } + + /** + * Given an actual and an expected iteration count, determine a single metric to report. + */ + private int coalesceLoopCounts(Integer actual, Integer expected) { + if (expected == null || expected <= 0) { + return -1; + } else if (actual == null) { + return expected; + } else { + return actual; + } + } + + /** + * Report run metrics by creating an empty test run to stick them in + * <p /> + * Exposed for unit testing + */ + void reportMetrics(ITestInvocationListener listener, TestInfo test, + Map<String, String> metrics) { + // Create an empty testRun to report the parsed runMetrics + Log.e(LOG_TAG, String.format("About to report metrics for %s: %s", test.mTestMetricsName, + metrics)); + listener.testRunStarted(test.mTestMetricsName, 0); + listener.testRunEnded(0, metrics); + } + + @Override + public void setDevice(ITestDevice device) { + mTestDevice = device; + } + + @Override + public ITestDevice getDevice() { + return mTestDevice; + } + + /** + * A meta-test to ensure that bits of the BluetoothStressTest are working properly + */ + public static class MetaTest extends TestCase { + private CameraStressTest mTestInstance = null; + + private TestInfo mTestInfo = null; + + private TestInfo mReportedTestInfo = null; + private Map<String, String> mReportedMetrics = null; + + private static String join(String... pieces) { + StringBuilder sb = new StringBuilder(); + for (String piece : pieces) { + sb.append(piece); + sb.append("\n"); + } + return sb.toString(); + } + + @Override + public void setUp() throws Exception { + mTestInstance = new CameraStressTest() { + @Override + void reportMetrics(ITestInvocationListener l, TestInfo test, + Map<String, String> metrics) { + mReportedTestInfo = test; + mReportedMetrics = metrics; + } + }; + + // Image capture stress test + mTestInfo = new TestInfo(); + TestInfo t = mTestInfo; // for convenience + t.mTestName = "capture"; + t.mClassName = "com.android.camera.stress.ImageCapture"; + t.mTestMetricsName = "camera_application_stress"; + t.mPatternMap = getPatternMap(); + } + + /** + * Make sure that parsing works for devices sending output in the old format + */ + public void testParse_old() throws Exception { + String output = join( + "Camera Image Capture", + "No of loops :100", + "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + + ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 " + + ",36 ,37 ,38 ,39 ,40 ,41 ,42", + "Camera Video Capture", + "No of loops :100", + "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + + ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 " + + ",36 ,37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 " + + ",53 ,54 ,55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 " + + ",70 ,71 ,72 ,73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 " + + ",87 ,88 ,89 ,90 ,91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99", + "Camera Switch Mode:", + "No of loops :200", + "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13"); + + InputStream iStream = new ByteArrayInputStream(output.getBytes()); + mTestInstance.parseOutputFile(mTestInfo, iStream, null); + assertEquals(mTestInfo, mReportedTestInfo); + assertNotNull(mReportedMetrics); + Log.e(LOG_TAG, String.format("Got reported metrics: %s", mReportedMetrics.toString())); + assertEquals(3, mReportedMetrics.size()); + assertEquals("43", mReportedMetrics.get("ImageCapture")); + assertEquals("100", mReportedMetrics.get("VideoRecording")); + assertEquals("14", mReportedMetrics.get("SwitchPreview")); + } + + /** + * Make sure that parsing works for devices sending output in the new format + */ + public void testParse_new() throws Exception { + String output = join( + "Camera Stress Test result", + "/folder/subfolder/data/CameraStressTest_git_honeycomb-mr1-release_" + + "1700614441c02617_109535_CameraStressOut.txt", + "Back Camera Image Capture", + "No of loops :100", + "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + + ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " + + ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " + + ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " + + ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " + + ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99", + "Front Camera Image Capture", + "No of loops :100", + "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + + ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " + + ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " + + ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " + + ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " + + ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98", + "Back Camera Video Capture", + "No of loops :100", + "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + + ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " + + ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " + + ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " + + ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " + + ",91 ,92 ,93 ,94 ,95 ,96 ,97", + "Front Camera Video Capture", + "No of loops :100", + "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + + ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " + + ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " + + ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " + + ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " + + ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99", + "Camera Switch Mode:", + "No of loops :200", + "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + + ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " + + ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " + + ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " + + ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " + + ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99 ,100 ,101 ,102 ,103 ,104 ,105 ,106 " + + ",107 ,108 ,109 ,110 ,111 ,112 ,113 ,114 ,115 ,116 ,117 ,118 ,119 ,120 " + + ",121 ,122 ,123 ,124 ,125 ,126 ,127 ,128 ,129 ,130 ,131 ,132 ,133 ,134 " + + ",135 ,136 ,137 ,138 ,139 ,140 ,141 ,142 ,143 ,144 ,145 ,146 ,147 ,148 " + + ",149 ,150 ,151 ,152 ,153 ,154 ,155 ,156 ,157 ,158 ,159 ,160 ,161 ,162 " + + ",163 ,164 ,165 ,166 ,167 ,168 ,169 ,170 ,171 ,172 ,173 ,174 ,175 ,176 " + + ",177 ,178 ,179 ,180 ,181 ,182 ,183 ,184 ,185 ,186 ,187 ,188 ,189 ,190 " + + ",191 ,192 ,193 ,194 ,195 ,196 ,197 ,198 ,199"); + + InputStream iStream = new ByteArrayInputStream(output.getBytes()); + mTestInstance.parseOutputFile(mTestInfo, iStream, null); + assertEquals(mTestInfo, mReportedTestInfo); + assertNotNull(mReportedMetrics); + Log.e(LOG_TAG, String.format("Got reported metrics: %s", mReportedMetrics.toString())); + assertEquals(5, mReportedMetrics.size()); + assertEquals("100", mReportedMetrics.get("ImageCapture")); + assertEquals("99", mReportedMetrics.get("FrontImageCapture")); + assertEquals("98", mReportedMetrics.get("VideoRecording")); + assertEquals("100", mReportedMetrics.get("FrontVideoRecording")); + assertEquals("200", mReportedMetrics.get("SwitchPreview")); + } + } +} + diff --git a/src/com/android/media/tests/CameraTestBase.java b/src/com/android/media/tests/CameraTestBase.java new file mode 100644 index 0000000..8714a22 --- /dev/null +++ b/src/com/android/media/tests/CameraTestBase.java @@ -0,0 +1,840 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.CollectingOutputReceiver; +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.IConfiguration; +import com.android.tradefed.config.IConfigurationReceiver; +import com.android.tradefed.config.Option; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ByteArrayInputStreamSource; +import com.android.tradefed.result.CollectingTestListener; +import com.android.tradefed.result.FileInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.testtype.InstrumentationTest; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.IRunUtil; +import com.android.tradefed.util.RunUtil; +import com.android.tradefed.util.StreamUtil; + +import org.junit.Assert; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.TimeUnit; + +/** + * Camera test base class + * + * Camera2StressTest, CameraStartupTest, Camera2LatencyTest and CameraPerformanceTest use this base + * class for Camera ivvavik and later. + */ +public class CameraTestBase implements IDeviceTest, IRemoteTest, IConfigurationReceiver { + + private static final long SHELL_TIMEOUT_MS = 60 * 1000; // 1 min + private static final int SHELL_MAX_ATTEMPTS = 3; + protected static final String PROCESS_CAMERA_DAEMON = "mm-qcamera-daemon"; + protected static final String PROCESS_MEDIASERVER = "mediaserver"; + protected static final String PROCESS_CAMERA_APP = "com.google.android.GoogleCamera"; + protected static final String DUMP_ION_HEAPS_COMMAND = "cat /d/ion/heaps/system"; + protected static final String ARGUMENT_TEST_ITERATIONS = "iterations"; + + @Option(name = "test-package", description = "Test package to run.") + private String mTestPackage = "com.google.android.camera"; + + @Option(name = "test-class", description = "Test class to run.") + private String mTestClass = null; + + @Option(name = "test-methods", description = "Test method to run. May be repeated.") + private Collection<String> mTestMethods = new ArrayList<>(); + + @Option(name = "test-runner", description = "Test runner for test instrumentation.") + private String mTestRunner = "android.test.InstrumentationTestRunner"; + + @Option(name = "test-timeout", description = "Max time allowed in ms for a test run.") + private int mTestTimeoutMs = 60 * 60 * 1000; // 1 hour + + @Option(name = "shell-timeout", + description="The defined timeout (in milliseconds) is used as a maximum waiting time " + + "when expecting the command output from the device. At any time, if the " + + "shell command does not output anything for a period longer than defined " + + "timeout the TF run terminates. For no timeout, set to 0.") + private long mShellTimeoutMs = 60 * 60 * 1000; // default to 1 hour + + @Option(name = "ru-key", description = "Result key to use when posting to the dashboard.") + private String mRuKey = null; + + @Option(name = "logcat-on-failure", description = + "take a logcat snapshot on every test failure.") + private boolean mLogcatOnFailure = false; + + @Option( + name = "instrumentation-arg", + description = "Additional instrumentation arguments to provide." + ) + private Map<String, String> mInstrArgMap = new HashMap<>(); + + @Option(name = "dump-meminfo", description = + "take a dumpsys meminfo at a given interval time.") + private boolean mDumpMeminfo = false; + + @Option(name="dump-meminfo-interval-ms", + description="Interval of calling dumpsys meminfo in milliseconds.") + private int mMeminfoIntervalMs = 5 * 60 * 1000; // 5 minutes + + @Option(name = "dump-ion-heap", description = + "dump ION allocations at the end of test.") + private boolean mDumpIonHeap = false; + + @Option(name = "dump-thread-count", description = + "Count the number of threads in Camera process at a given interval time.") + private boolean mDumpThreadCount = false; + + @Option(name="dump-thread-count-interval-ms", + description="Interval of calling ps to count the number of threads in milliseconds.") + private int mThreadCountIntervalMs = 5 * 60 * 1000; // 5 minutes + + @Option(name="iterations", description="The number of iterations to run. Default to 1. " + + "This takes effect only when Camera2InstrumentationTestRunner is used to execute " + + "framework stress tests.") + private int mIterations = 1; + + private ITestDevice mDevice = null; + + // A base listener to collect the results from each test run. Test results will be forwarded + // to other listeners. + private AbstractCollectingListener mCollectingListener = null; + + private long mStartTimeMs = 0; + + private MeminfoTimer mMeminfoTimer = null; + private ThreadTrackerTimer mThreadTrackerTimer = null; + + protected IConfiguration mConfiguration; + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + // ignore + } + + /** + * Run Camera instrumentation test with a default listener. + * + * @param listener the ITestInvocationListener of test results + * @throws DeviceNotAvailableException + */ + protected void runInstrumentationTest(ITestInvocationListener listener) + throws DeviceNotAvailableException { + if (mCollectingListener == null) { + mCollectingListener = new DefaultCollectingListener(listener); + } + runInstrumentationTest(listener, mCollectingListener); + } + + /** + * Run Camera instrumentation test with a listener to gather the metrics from the individual + * test runs. + * + * @param listener the ITestInvocationListener of test results + * @param collectingListener the {@link CollectingTestListener} to collect the metrics from + * test runs + * @throws DeviceNotAvailableException + */ + protected void runInstrumentationTest(ITestInvocationListener listener, + AbstractCollectingListener collectingListener) + throws DeviceNotAvailableException { + Assert.assertNotNull(collectingListener); + mCollectingListener = collectingListener; + + InstrumentationTest instr = new InstrumentationTest(); + instr.setDevice(getDevice()); + instr.setPackageName(getTestPackage()); + instr.setRunnerName(getTestRunner()); + instr.setClassName(getTestClass()); + instr.setTestTimeout(getTestTimeoutMs()); + instr.setShellTimeout(getShellTimeoutMs()); + instr.setLogcatOnFailure(mLogcatOnFailure); + instr.setRunName(getRuKey()); + instr.setRerunMode(false); + + // Set test iteration. + if (getIterationCount() > 1) { + CLog.v("Setting test iterations: %d", getIterationCount()); + Map<String, String> instrArgMap = getInstrumentationArgMap(); + instrArgMap.put(ARGUMENT_TEST_ITERATIONS, String.valueOf(getIterationCount())); + } + + for (Map.Entry<String, String> entry : getInstrumentationArgMap().entrySet()) { + instr.addInstrumentationArg(entry.getKey(), entry.getValue()); + } + + // Check if dumpheap needs to be taken for native processes before test runs. + if (shouldDumpMeminfo()) { + mMeminfoTimer = new MeminfoTimer(0, mMeminfoIntervalMs); + } + if (shouldDumpThreadCount()) { + long delayMs = mThreadCountIntervalMs / 2; // Not to run all dump at the same interval. + mThreadTrackerTimer = new ThreadTrackerTimer(delayMs, mThreadCountIntervalMs); + } + + // Run tests. + mStartTimeMs = System.currentTimeMillis(); + if (mTestMethods.size() > 0) { + CLog.d(String.format("The number of test methods is: %d", mTestMethods.size())); + for (String testName : mTestMethods) { + instr.setMethodName(testName); + instr.run(mCollectingListener); + } + } else { + instr.run(mCollectingListener); + } + + dumpIonHeaps(mCollectingListener, getTestClass()); + } + + /** + * A base listener to collect all test results and metrics from Camera instrumentation test run. + * Abstract methods can be overridden to handle test metrics or inform of test run ended. + */ + protected abstract class AbstractCollectingListener extends CollectingTestListener { + + private ITestInvocationListener mListener = null; + private Map<String, String> mMetrics = new HashMap<>(); + private Map<String, String> mFatalErrors = new HashMap<>(); + + private static final String INCOMPLETE_TEST_ERR_MSG_PREFIX = + "Test failed to run to completion. Reason: 'Instrumentation run failed"; + + public AbstractCollectingListener(ITestInvocationListener listener) { + mListener = listener; + } + + /** + * Override only when subclasses need to get the test metrics from an individual + * instrumentation test. To aggregate the metrics from each test, update the + * getAggregatedMetrics and post them at the end of test run. + * + * @param test identifies the test + * @param testMetrics a {@link Map} of the metrics emitted + */ + abstract public void handleMetricsOnTestEnded(TestIdentifier test, + Map<String, String> testMetrics); + + /** + * Override only when it needs to inform subclasses of instrumentation test run ended, + * so that subclasses have a chance to peek the aggregated results at the end of test run + * and to decide what metrics to be posted. + * Either {@link ITestInvocationListener#testRunEnded} or + * {@link ITestInvocationListener#testRunFailed} should be called in this function to + * report the test run status. + * + * @param listener - the ITestInvocationListener of test results + * @param elapsedTime - device reported elapsed time, in milliseconds + * @param runMetrics - key-value pairs reported at the end of an instrumentation test run. + * Use getAggregatedMetrics to retrieve the metrics aggregated + * from an individual test, instead. + */ + abstract public void handleTestRunEnded(ITestInvocationListener listener, + long elapsedTime, Map<String, String> runMetrics); + + /** + * Report the end of an individual camera test and delegate handling the collected metrics + * to subclasses. Do not override testEnded to manipulate the test metrics after each test. + * Instead, use handleMetricsOnTestEnded. + * + * @param test identifies the test + * @param testMetrics a {@link Map} of the metrics emitted + */ + @Override + public void testEnded(TestIdentifier test, long endTime, Map<String, String> testMetrics) { + super.testEnded(test, endTime, testMetrics); + handleMetricsOnTestEnded(test, testMetrics); + stopDumping(test); + mListener.testEnded(test, endTime, testMetrics); + } + + @Override + public void testStarted(TestIdentifier test, long startTime) { + super.testStarted(test, startTime); + startDumping(test); + mListener.testStarted(test, startTime); + } + + @Override + public void testFailed(TestIdentifier test, String trace) { + super.testFailed(test, trace); + // If the test failed to run to complete, this is an exceptional case. + // Let this test run fail so that it can rerun. + if (trace.startsWith(INCOMPLETE_TEST_ERR_MSG_PREFIX)) { + mFatalErrors.put(test.getTestName(), trace); + CLog.d("Test (%s) failed due to fatal error : %s", test.getTestName(), trace); + } + mListener.testFailed(test, trace); + } + + @Override + public void testRunFailed(String errorMessage) { + super.testRunFailed(errorMessage); + mFatalErrors.put(getRuKey(), errorMessage); + } + + @Override + public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) { + super.testRunEnded(elapsedTime, runMetrics); + handleTestRunEnded(mListener, elapsedTime, runMetrics); + // never be called since handleTestRunEnded will handle it if needed. + //mListener.testRunEnded(elapsedTime, runMetrics); + } + + @Override + public void testRunStarted(String runName, int testCount) { + super.testRunStarted(runName, testCount); + mListener.testRunStarted(runName, testCount); + } + + @Override + public void testRunStopped(long elapsedTime) { + super.testRunStopped(elapsedTime); + mListener.testRunStopped(elapsedTime); + } + + @Override + public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) { + super.testLog(dataName, dataType, dataStream); + mListener.testLog(dataName, dataType, dataStream); + } + + protected void startDumping(TestIdentifier test) { + if (shouldDumpMeminfo()) { + mMeminfoTimer.start(test); + } + if (shouldDumpThreadCount()) { + mThreadTrackerTimer.start(test); + } + } + + protected void stopDumping(TestIdentifier test) { + InputStreamSource outputSource = null; + File outputFile = null; + if (shouldDumpMeminfo()) { + mMeminfoTimer.stop(); + // Grab a snapshot of meminfo file and post it to dashboard. + try { + outputFile = mMeminfoTimer.getOutputFile(); + outputSource = new FileInputStreamSource(outputFile, true /* delete */); + String logName = String.format("meminfo_%s", test.getTestName()); + mListener.testLog(logName, LogDataType.TEXT, outputSource); + } finally { + StreamUtil.cancel(outputSource); + } + } + if (shouldDumpThreadCount()) { + mThreadTrackerTimer.stop(); + try { + outputFile = mThreadTrackerTimer.getOutputFile(); + outputSource = new FileInputStreamSource(outputFile, true /* delete */); + String logName = String.format("ps_%s", test.getTestName()); + mListener.testLog(logName, LogDataType.TEXT, outputSource); + } finally { + StreamUtil.cancel(outputSource); + } + } + } + + public Map<String, String> getAggregatedMetrics() { + return mMetrics; + } + + public ITestInvocationListener getListeners() { + return mListener; + } + + /** + * Determine that the test run failed with fatal errors. + * + * @return True if test run has a failure due to fatal error. + */ + public boolean hasTestRunFatalError() { + return (getNumTotalTests() > 0 && mFatalErrors.size() > 0); + } + + public Map<String, String> getFatalErrors() { + return mFatalErrors; + } + + public String getErrorMessage() { + StringBuilder sb = new StringBuilder(); + for (Map.Entry<String, String> error : mFatalErrors.entrySet()) { + sb.append(error.getKey()); + sb.append(" : "); + sb.append(error.getValue()); + sb.append("\n"); + } + return sb.toString(); + } + } + + protected class DefaultCollectingListener extends AbstractCollectingListener { + + public DefaultCollectingListener(ITestInvocationListener listener) { + super(listener); + } + + @Override + public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) { + if (testMetrics == null) { + return; // No-op if there is nothing to post. + } + getAggregatedMetrics().putAll(testMetrics); + } + + @Override + public void handleTestRunEnded(ITestInvocationListener listener, long elapsedTime, + Map<String, String> runMetrics) { + // Post aggregated metrics at the end of test run. + listener.testRunEnded(getTestDurationMs(), getAggregatedMetrics()); + } + } + + // TODO: Leverage AUPT to collect system logs (meminfo, ION allocations and processes/threads) + private class MeminfoTimer { + + private static final String LOG_HEADER = + "uptime,pssCameraDaemon,pssCameraApp,ramTotal,ramFree,ramUsed"; + private static final String DUMPSYS_MEMINFO_COMMAND = + "dumpsys meminfo -c | grep -w -e " + "^ram -e ^time"; + private String[] mDumpsysMemInfoProc = { + PROCESS_CAMERA_DAEMON, PROCESS_CAMERA_APP, PROCESS_MEDIASERVER + }; + private static final int STATE_STOPPED = 0; + private static final int STATE_SCHEDULED = 1; + private static final int STATE_RUNNING = 2; + + private int mState = STATE_STOPPED; + private Timer mTimer = new Timer(true); // run as a daemon thread + private long mDelayMs = 0; + private long mPeriodMs = 60 * 1000; // 60 sec + private File mOutputFile = null; + private String mCommand; + + public MeminfoTimer(long delayMs, long periodMs) { + mDelayMs = delayMs; + mPeriodMs = periodMs; + mCommand = DUMPSYS_MEMINFO_COMMAND; + for (String process : mDumpsysMemInfoProc) { + mCommand += " -e " + process; + } + } + + synchronized void start(TestIdentifier test) { + if (isRunning()) { + stop(); + } + // Create an output file. + if (createOutputFile(test) == null) { + CLog.w("Stop collecting meminfo since meminfo log file not found."); + mState = STATE_STOPPED; + return; // No-op + } + mTimer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + mState = STATE_RUNNING; + dumpMeminfo(mCommand, mOutputFile); + } + }, mDelayMs, mPeriodMs); + mState = STATE_SCHEDULED; + } + + synchronized void stop() { + mState = STATE_STOPPED; + mTimer.cancel(); + } + + synchronized boolean isRunning() { + return (mState == STATE_RUNNING); + } + + public File getOutputFile() { + return mOutputFile; + } + + private File createOutputFile(TestIdentifier test) { + try { + mOutputFile = FileUtil.createTempFile( + String.format("meminfo_%s", test.getTestName()), "csv"); + BufferedWriter writer = new BufferedWriter(new FileWriter(mOutputFile, false)); + writer.write(LOG_HEADER); + writer.newLine(); + writer.flush(); + writer.close(); + } catch (IOException e) { + CLog.w("Failed to create meminfo log file %s:", mOutputFile.getAbsolutePath()); + CLog.e(e); + return null; + } + return mOutputFile; + } + } + + void dumpMeminfo(String command, File outputFile) { + try { + CollectingOutputReceiver receiver = new CollectingOutputReceiver(); + // Dump meminfo in a compact form. + getDevice().executeShellCommand(command, receiver, + SHELL_TIMEOUT_MS, TimeUnit.MILLISECONDS, SHELL_MAX_ATTEMPTS); + printMeminfo(outputFile, receiver.getOutput()); + } catch (DeviceNotAvailableException e) { + CLog.w("Failed to dump meminfo:"); + CLog.e(e); + } + } + + void printMeminfo(File outputFile, String meminfo) { + // Parse meminfo and print each iteration in a line in a .csv format. The meminfo output + // are separated into three different formats: + // + // Format: time,<uptime>,<realtime> + // eg. "time,59459911,63354673" + // + // Format: proc,<oom_label>,<process_name>,<pid>,<pss>,<hasActivities> + // eg. "proc,native,mm-qcamera-daemon,522,12881,e" + // "proc,fore,com.google.android.GoogleCamera,26560,70880,a" + // + // Format: ram,<total>,<free>,<used> + // eg. "ram,1857364,810189,541516" + BufferedWriter writer = null; + BufferedReader reader = null; + try { + final String DELIMITER = ","; + writer = new BufferedWriter(new FileWriter(outputFile, true)); + reader = new BufferedReader(new StringReader(meminfo)); + String line; + String uptime = null; + String pssCameraNative = null; + String pssCameraApp = null; + String ramTotal = null; + String ramFree = null; + String ramUsed = null; + while ((line = reader.readLine()) != null) { + if (line.startsWith("time")) { + uptime = line.split(DELIMITER)[1]; + } else if (line.startsWith("ram")) { + String[] ram = line.split(DELIMITER); + ramTotal = ram[1]; + ramFree = ram[2]; + ramUsed = ram[3]; + } else if (line.contains(PROCESS_CAMERA_DAEMON)) { + pssCameraNative = line.split(DELIMITER)[4]; + } else if (line.contains(PROCESS_CAMERA_APP)) { + pssCameraApp = line.split(DELIMITER)[4]; + } + } + String printMsg = String.format( + "%s,%s,%s,%s,%s,%s", uptime, pssCameraNative, pssCameraApp, + ramTotal, ramFree, ramUsed); + writer.write(printMsg); + writer.newLine(); + writer.flush(); + } catch (IOException e) { + CLog.w("Failed to print meminfo to %s:", outputFile.getAbsolutePath()); + CLog.e(e); + } finally { + StreamUtil.close(writer); + } + } + + // TODO: Leverage AUPT to collect system logs (meminfo, ION allocations and processes/threads) + private class ThreadTrackerTimer { + + // list all threads in a given process, remove the first header line, squeeze whitespaces, + // select thread name (in 14th column), then sort and group by its name. + // Examples: + // 3 SoundPoolThread + // 3 SoundPool + // 2 Camera Job Disp + // 1 pool-7-thread-1 + // 1 pool-6-thread-1 + // FIXME: Resolve the error "sh: syntax error: '|' unexpected" using the command below + // $ /system/bin/ps -t -p %s | tr -s ' ' | cut -d' ' -f13- | sort | uniq -c | sort -nr" + private static final String PS_COMMAND_FORMAT = "/system/bin/ps -t -p %s"; + private static final String PGREP_COMMAND_FORMAT = "pgrep %s"; + private static final int STATE_STOPPED = 0; + private static final int STATE_SCHEDULED = 1; + private static final int STATE_RUNNING = 2; + + private int mState = STATE_STOPPED; + private Timer mTimer = new Timer(true); // run as a daemon thread + private long mDelayMs = 0; + private long mPeriodMs = 60 * 1000; // 60 sec + private File mOutputFile = null; + + public ThreadTrackerTimer(long delayMs, long periodMs) { + mDelayMs = delayMs; + mPeriodMs = periodMs; + } + + synchronized void start(TestIdentifier test) { + if (isRunning()) { + stop(); + } + // Create an output file. + if (createOutputFile(test) == null) { + CLog.w("Stop collecting thread counts since log file not found."); + mState = STATE_STOPPED; + return; // No-op + } + mTimer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + mState = STATE_RUNNING; + dumpThreadCount(PS_COMMAND_FORMAT, getPid(PROCESS_CAMERA_APP), mOutputFile); + } + }, mDelayMs, mPeriodMs); + mState = STATE_SCHEDULED; + } + + synchronized void stop() { + mState = STATE_STOPPED; + mTimer.cancel(); + } + + synchronized boolean isRunning() { + return (mState == STATE_RUNNING); + } + + public File getOutputFile() { + return mOutputFile; + } + + File createOutputFile(TestIdentifier test) { + try { + mOutputFile = FileUtil.createTempFile( + String.format("ps_%s", test.getTestName()), "txt"); + new BufferedWriter(new FileWriter(mOutputFile, false)).close(); + } catch (IOException e) { + CLog.w("Failed to create processes and threads file %s:", + mOutputFile.getAbsolutePath()); + CLog.e(e); + return null; + } + return mOutputFile; + } + + String getPid(String processName) { + String result = null; + try { + result = getDevice().executeShellCommand(String.format(PGREP_COMMAND_FORMAT, + processName)); + } catch (DeviceNotAvailableException e) { + CLog.w("Failed to get pid %s:", processName); + CLog.e(e); + } + return result; + } + + String getUptime() { + String uptime = null; + try { + // uptime will typically have a format like "5278.73 1866.80". Use the first one + // (which is wall-time) + uptime = getDevice().executeShellCommand("cat /proc/uptime").split(" ")[0]; + Float.parseFloat(uptime); + } catch (NumberFormatException e) { + CLog.w("Failed to get valid uptime %s: %s", uptime, e); + } catch (DeviceNotAvailableException e) { + CLog.w("Failed to get valid uptime: %s", e); + } + return uptime; + } + + void dumpThreadCount(String commandFormat, String pid, File outputFile) { + try { + if ("".equals(pid)) { + return; + } + String result = getDevice().executeShellCommand(String.format(commandFormat, pid)); + String header = String.format("UPTIME: %s", getUptime()); + BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile, true)); + writer.write(header); + writer.newLine(); + writer.write(result); + writer.newLine(); + writer.flush(); + writer.close(); + } catch (DeviceNotAvailableException | IOException e) { + CLog.w("Failed to dump thread count:"); + CLog.e(e); + } + } + } + + // TODO: Leverage AUPT to collect system logs (meminfo, ION allocations and + // processes/threads) + protected void dumpIonHeaps(ITestInvocationListener listener, String testClass) { + if (!shouldDumpIonHeap()) { + return; // No-op if option is not set. + } + try { + String result = getDevice().executeShellCommand(DUMP_ION_HEAPS_COMMAND); + if (!"".equals(result)) { + String fileName = String.format("ionheaps_%s_onEnd", testClass); + listener.testLog(fileName, LogDataType.TEXT, + new ByteArrayInputStreamSource(result.getBytes())); + } + } catch (DeviceNotAvailableException e) { + CLog.w("Failed to dump ION heaps:"); + CLog.e(e); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void setDevice(ITestDevice device) { + mDevice = device; + } + + /** + * {@inheritDoc} + */ + @Override + public ITestDevice getDevice() { + return mDevice; + } + + /** + * {@inheritDoc} + */ + @Override + public void setConfiguration(IConfiguration configuration) { + mConfiguration = configuration; + } + + /** + * Get the {@link IRunUtil} instance to use. + * <p/> + * Exposed so unit tests can mock. + */ + IRunUtil getRunUtil() { + return RunUtil.getDefault(); + } + + /** + * Get the duration of Camera test instrumentation in milliseconds. + * + * @return the duration of Camera instrumentation test until it is called. + */ + public long getTestDurationMs() { + return System.currentTimeMillis() - mStartTimeMs; + } + + public String getTestPackage() { + return mTestPackage; + } + + public void setTestPackage(String testPackage) { + mTestPackage = testPackage; + } + + public String getTestClass() { + return mTestClass; + } + + public void setTestClass(String testClass) { + mTestClass = testClass; + } + + public String getTestRunner() { + return mTestRunner; + } + + public void setTestRunner(String testRunner) { + mTestRunner = testRunner; + } + + public int getTestTimeoutMs() { + return mTestTimeoutMs; + } + + public void setTestTimeoutMs(int testTimeoutMs) { + mTestTimeoutMs = testTimeoutMs; + } + + public long getShellTimeoutMs() { + return mShellTimeoutMs; + } + + public void setShellTimeoutMs(long shellTimeoutMs) { + mShellTimeoutMs = shellTimeoutMs; + } + + public String getRuKey() { + return mRuKey; + } + + public void setRuKey(String ruKey) { + mRuKey = ruKey; + } + + public boolean shouldDumpMeminfo() { + return mDumpMeminfo; + } + + public boolean shouldDumpIonHeap() { + return mDumpIonHeap; + } + + public boolean shouldDumpThreadCount() { + return mDumpThreadCount; + } + + public AbstractCollectingListener getCollectingListener() { + return mCollectingListener; + } + + public void setLogcatOnFailure(boolean logcatOnFailure) { + mLogcatOnFailure = logcatOnFailure; + } + + public int getIterationCount() { + return mIterations; + } + + public Map<String, String> getInstrumentationArgMap() { return mInstrArgMap; } +} diff --git a/src/com/android/media/tests/MediaMemoryTest.java b/src/com/android/media/tests/MediaMemoryTest.java new file mode 100644 index 0000000..bb09488 --- /dev/null +++ b/src/com/android/media/tests/MediaMemoryTest.java @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.IDevice; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.android.tradefed.config.Option; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.BugreportCollector; +import com.android.tradefed.result.BugreportCollector.Freq; +import com.android.tradefed.result.BugreportCollector.Noun; +import com.android.tradefed.result.BugreportCollector.Relation; +import com.android.tradefed.result.FileInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.StreamUtil; + +import org.junit.Assert; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Runs the Media memory test. This test will do various media actions ( ie. + * playback, recording and etc.) then capture the snapshot of mediaserver memory + * usage. The test summary is save to /sdcard/mediaMemOutput.txt + * <p/> + * Note that this test will not run properly unless /sdcard is mounted and + * writable. + */ +public class MediaMemoryTest implements IDeviceTest, IRemoteTest { + + ITestDevice mTestDevice = null; + + private static final String METRICS_RUN_NAME = "MediaMemoryLeak"; + + // Constants for running the tests + private static final String TEST_CLASS_NAME = + "com.android.mediaframeworktest.performance.MediaPlayerPerformance"; + private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest"; + private static final String TEST_RUNNER_NAME = ".MediaFrameworkPerfTestRunner"; + + private final String mOutputPaths[] = {"mediaMemOutput.txt","mediaProcmemOutput.txt"}; + + //Max test timeout - 4 hrs + private static final int MAX_TEST_TIMEOUT = 4 * 60 * 60 * 1000; + + public Map<String, String> mPatternMap = new HashMap<>(); + private static final Pattern TOTAL_MEM_DIFF_PATTERN = + Pattern.compile("^The total diff = (\\d+)"); + + @Option(name = "getHeapDump", description = "Collect the heap ") + private boolean mGetHeapDump = false; + + @Option(name = "getProcMem", description = "Collect the procmem info ") + private boolean mGetProcMem = false; + + @Option(name = "testName", description = "Test name to run. May be repeated.") + private Collection<String> mTests = new LinkedList<>(); + + public MediaMemoryTest() { + mPatternMap.put("testCameraPreviewMemoryUsage", "CameraPreview"); + mPatternMap.put("testRecordAudioOnlyMemoryUsage", "AudioRecord"); + mPatternMap.put("testH263VideoPlaybackMemoryUsage", "H263Playback"); + mPatternMap.put("testRecordVideoAudioMemoryUsage", "H263RecordVideoAudio"); + mPatternMap.put("testH263RecordVideoOnlyMemoryUsage", "H263RecordVideoOnly"); + mPatternMap.put("testH264VideoPlaybackMemoryUsage", "H264Playback"); + mPatternMap.put("testMpeg4RecordVideoOnlyMemoryUsage", "MPEG4RecordVideoOnly"); + } + + + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + Assert.assertNotNull(mTestDevice); + + IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, + TEST_RUNNER_NAME, mTestDevice.getIDevice()); + runner.setClassName(TEST_CLASS_NAME); + runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS); + if (mGetHeapDump) { + runner.addInstrumentationArg("get_heap_dump", "true"); + } + if (mGetProcMem) { + runner.addInstrumentationArg("get_procmem", "true"); + } + + BugreportCollector bugListener = new BugreportCollector(listener, + mTestDevice); + bugListener.addPredicate(new BugreportCollector.Predicate( + Relation.AFTER, Freq.EACH, Noun.TESTRUN)); + + if (mTests.size() > 0) { + for (String testName : mTests) { + runner.setMethodName(TEST_CLASS_NAME, testName); + mTestDevice.runInstrumentationTests(runner, bugListener); + } + } else { + mTestDevice.runInstrumentationTests(runner, bugListener); + } + logOutputFiles(listener); + cleanResultFile(); + } + + /** + * Clean up the test result file from test run + */ + private void cleanResultFile() throws DeviceNotAvailableException { + String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + for(String outputPath : mOutputPaths){ + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, outputPath)); + } + if (mGetHeapDump) { + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, "*.dump")); + } + } + + private void uploadHeapDumpFiles(ITestInvocationListener listener) + throws DeviceNotAvailableException { + // Pull and upload the heap dump output files. + InputStreamSource outputSource = null; + File outputFile = null; + + String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + + String out = mTestDevice.executeShellCommand(String.format("ls %s/%s", + extStore, "*.dump")); + String heapOutputFiles[] = out.split("\n"); + + for (String heapFile : heapOutputFiles) { + try { + outputFile = mTestDevice.pullFile(heapFile.trim()); + if (outputFile == null) { + continue; + } + outputSource = new FileInputStreamSource(outputFile); + listener.testLog(heapFile, LogDataType.TEXT, outputSource); + } finally { + FileUtil.deleteFile(outputFile); + StreamUtil.cancel(outputSource); + } + } + } + + /** + * Pull the output files from the device, add it to the logs, and also parse + * out the relevant test metrics and report them. + */ + private void logOutputFiles(ITestInvocationListener listener) + throws DeviceNotAvailableException { + File outputFile = null; + InputStreamSource outputSource = null; + + if (mGetHeapDump) { + // Upload all the heap dump files. + uploadHeapDumpFiles(listener); + } + for(String outputPath : mOutputPaths){ + try { + outputFile = mTestDevice.pullFileFromExternal(outputPath); + + if (outputFile == null) { + return; + } + + // Upload a verbatim copy of the output file + CLog.d("Sending %d byte file %s into the logosphere!", + outputFile.length(), outputFile); + outputSource = new FileInputStreamSource(outputFile); + listener.testLog(outputPath, LogDataType.TEXT, outputSource); + + // Parse the output file to upload aggregated metrics + parseOutputFile(new FileInputStream(outputFile), listener); + } catch (IOException e) { + CLog.e("IOException while reading or parsing output file: %s", + e.getMessage()); + } finally { + FileUtil.deleteFile(outputFile); + StreamUtil.cancel(outputSource); + } + } + } + + /** + * Parse the relevant metrics from the Instrumentation test output file + */ + private void parseOutputFile(InputStream dataStream, + ITestInvocationListener listener) { + + Map<String, String> runMetrics = new HashMap<>(); + + // try to parse it + String contents; + try { + contents = StreamUtil.getStringFromStream(dataStream); + } catch (IOException e) { + CLog.e("Got IOException during test processing: %s", + e.getMessage()); + return; + } + + List<String> lines = Arrays.asList(contents.split("\n")); + ListIterator<String> lineIter = lines.listIterator(); + String line; + while (lineIter.hasNext()) { + line = lineIter.next(); + if (mPatternMap.containsKey(line)) { + + String key = mPatternMap.get(line); + // Look for the total diff + while (lineIter.hasNext()) { + line = lineIter.next(); + Matcher m = TOTAL_MEM_DIFF_PATTERN.matcher(line); + if (m.matches()) { + int result = Integer.parseInt(m.group(1)); + runMetrics.put(key, Integer.toString(result)); + break; + } + } + } else { + CLog.e("Got unmatched line: %s", line); + continue; + } + } + reportMetrics(listener, runMetrics); + } + + /** + * Report run metrics by creating an empty test run to stick them in + * <p /> + * Exposed for unit testing + */ + void reportMetrics(ITestInvocationListener listener, Map<String, String> metrics) { + CLog.d("About to report metrics: %s", metrics); + listener.testRunStarted(METRICS_RUN_NAME, 0); + listener.testRunEnded(0, metrics); + } + + @Override + public void setDevice(ITestDevice device) { + mTestDevice = device; + } + + @Override + public ITestDevice getDevice() { + return mTestDevice; + } +} diff --git a/src/com/android/media/tests/MediaPlayerStressTest.java b/src/com/android/media/tests/MediaPlayerStressTest.java new file mode 100644 index 0000000..8efab3e --- /dev/null +++ b/src/com/android/media/tests/MediaPlayerStressTest.java @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.IDevice; +import com.android.ddmlib.Log; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.android.tradefed.config.Option; +import com.android.tradefed.config.Option.Importance; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.result.BugreportCollector; +import com.android.tradefed.result.BugreportCollector.Freq; +import com.android.tradefed.result.BugreportCollector.Noun; +import com.android.tradefed.result.BugreportCollector.Relation; +import com.android.tradefed.result.FileInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.RegexTrie; +import com.android.tradefed.util.StreamUtil; + +import org.junit.Assert; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Runs the Media Player stress test. This test will play the video files under + * the /sdcard/samples folder and capture the video playback event statistics in + * a text file under /sdcard/PlaybackTestResult.txt + * <p/> + * Note that this test will not run properly unless /sdcard is mounted and + * writable. + */ +public class MediaPlayerStressTest implements IDeviceTest, IRemoteTest { + private static final String LOG_TAG = "MediaPlayerStress"; + + ITestDevice mTestDevice = null; + @Option(name = "test-class", importance = Importance.ALWAYS) + private String mTestClassName = + "com.android.mediaframeworktest.stress.MediaPlayerStressTest"; + @Option(name = "metrics-name", importance = Importance.ALWAYS) + private String mMetricsRunName = "MediaPlayerStress"; + @Option(name = "result-file", importance = Importance.ALWAYS) + private String mOutputPath = "PlaybackTestResult.txt"; + + //Max test timeout - 10 hrs + private static final int MAX_TEST_TIMEOUT = 10 * 60 * 60 * 1000; + + // Constants for running the tests + private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest"; + private static final String TEST_RUNNER_NAME = ".MediaPlayerStressTestRunner"; + + public RegexTrie<String> mPatternMap = new RegexTrie<>(); + + public MediaPlayerStressTest() { + mPatternMap.put("PlaybackPass", "^Total Complete: (\\d+)"); + mPatternMap.put("PlaybackCrash", "^Total Error: (\\d+)"); + mPatternMap.put("TrackLagging", "^Total Track Lagging: (\\d+)"); + mPatternMap.put("BadInterleave", "^Total Bad Interleaving: (\\d+)"); + mPatternMap.put("FailedToCompleteWithNoError", + "^Total Failed To Complete With No Error: (\\d+)"); + } + + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + Assert.assertNotNull(mTestDevice); + IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, + TEST_RUNNER_NAME, mTestDevice.getIDevice()); + runner.setClassName(mTestClassName); + runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS); + + BugreportCollector bugListener = new BugreportCollector(listener, + mTestDevice); + bugListener.addPredicate(BugreportCollector.AFTER_FAILED_TESTCASES); + bugListener.setDescriptiveName("media_player_stress_test"); + bugListener.addPredicate(new BugreportCollector.Predicate( + Relation.AFTER, Freq.EACH, Noun.TESTRUN)); + + mTestDevice.runInstrumentationTests(runner, bugListener); + + logOutputFile(listener); + cleanResultFile(); + } + + /** + * Clean up the test result file from test run + */ + private void cleanResultFile() throws DeviceNotAvailableException { + String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath)); + } + + /** + * Pull the output file from the device, add it to the logs, and also parse + * out the relevant test metrics and report them. + */ + private void logOutputFile(ITestInvocationListener listener) + throws DeviceNotAvailableException { + File outputFile = null; + InputStreamSource outputSource = null; + try { + outputFile = mTestDevice.pullFileFromExternal(mOutputPath); + + if (outputFile == null) { + return; + } + + // Upload a verbatim copy of the output file + Log.d(LOG_TAG, String.format("Sending %d byte file %s into the logosphere!", + outputFile.length(), outputFile)); + outputSource = new FileInputStreamSource(outputFile); + listener.testLog(mOutputPath, LogDataType.TEXT, outputSource); + // Parse the output file to upload aggregated metrics + parseOutputFile(new FileInputStream(outputFile), listener); + } catch (IOException e) { + Log.e(LOG_TAG, String.format( + "IOException while reading or parsing output file: %s", e)); + } finally { + FileUtil.deleteFile(outputFile); + StreamUtil.cancel(outputSource); + } + } + + /** + * Parse the relevant metrics from the Instrumentation test output file + */ + private void parseOutputFile(InputStream dataStream, + ITestInvocationListener listener) { + Map<String, String> runMetrics = new HashMap<>(); + + // try to parse it + String contents; + try { + contents = StreamUtil.getStringFromStream(dataStream); + } catch (IOException e) { + Log.e(LOG_TAG, String.format( + "Got IOException during test processing: %s", e)); + return; + } + + List<String> lines = Arrays.asList(contents.split("\n")); + ListIterator<String> lineIter = lines.listIterator(); + String line; + while (lineIter.hasNext()) { + line = lineIter.next(); + List<List<String>> capture = new ArrayList<>(1); + String key = mPatternMap.retrieve(capture, line); + if (key != null) { + Log.d(LOG_TAG, String.format("Got '%s' and captures '%s'", + key, capture.toString())); + } else if (line.isEmpty()) { + // ignore + continue; + } else { + Log.d(LOG_TAG, String.format("Got unmatched line: %s", line)); + continue; + } + runMetrics.put(key, capture.get(0).get(0)); + } + reportMetrics(listener, runMetrics); + } + + /** + * Report run metrics by creating an empty test run to stick them in + * <p /> + * Exposed for unit testing + */ + void reportMetrics(ITestInvocationListener listener, Map<String, String> metrics) { + Log.d(LOG_TAG, String.format("About to report metrics: %s", metrics)); + listener.testRunStarted(mMetricsRunName, 0); + listener.testRunEnded(0, metrics); + } + + @Override + public void setDevice(ITestDevice device) { + mTestDevice = device; + } + + @Override + public ITestDevice getDevice() { + return mTestDevice; + } +} diff --git a/src/com/android/media/tests/MediaResultReporter.java b/src/com/android/media/tests/MediaResultReporter.java new file mode 100644 index 0000000..0d348c3 --- /dev/null +++ b/src/com/android/media/tests/MediaResultReporter.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.Log; +import com.android.tradefed.config.Option; +import com.android.tradefed.result.EmailResultReporter; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.result.TestSummary; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.List; + +/** + * Media reporter that send the test summary through email. + */ +public class MediaResultReporter extends EmailResultReporter { + + private static final String LOG_TAG = "MediaResultReporter"; + + public StringBuilder mEmailBodyBuilder; + private String mSummaryUrl = ""; + + @Option(name = "log-name", description = "Name of the report that attach to email") + private String mLogName = null; + + public MediaResultReporter() { + mEmailBodyBuilder = new StringBuilder(); + } + + /** + * {@inheritDoc} + */ + @Override + public String generateEmailBody() { + return mEmailBodyBuilder.toString(); + } + + /** + * {@inheritDoc} + */ + @Override + public void putSummary(List<TestSummary> summaries) { + // Get the summary url + if (summaries.isEmpty()) { + return; + } + mSummaryUrl = summaries.get(0).getSummary().getString(); + mEmailBodyBuilder.append(mSummaryUrl); + } + + /** + * {@inheritDoc} + */ + @Override + public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) { + + char[] buf = new char[2048]; + + try { + //Attached the test summary to the email body + if (mLogName.equals(dataName)) { + Reader r = new InputStreamReader(dataStream.createInputStream()); + while (true) { + int n = r.read(buf); + if (n < 0) { + break; + } + mEmailBodyBuilder.append(buf, 0, n); + } + mEmailBodyBuilder.append('\n'); + } + } catch (IOException e) { + Log.w(LOG_TAG, String.format("Exception while parsing %s: %s", dataName, e)); + } + } +} diff --git a/src/com/android/media/tests/MediaStressTest.java b/src/com/android/media/tests/MediaStressTest.java new file mode 100644 index 0000000..189dd60 --- /dev/null +++ b/src/com/android/media/tests/MediaStressTest.java @@ -0,0 +1,225 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.IDevice; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.FileInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.StreamUtil; + +import org.junit.Assert; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Runs the Media stress testcases. + * FIXME: more details + * <p/> + * Note that this test will not run properly unless /sdcard is mounted and writable. + */ +public class MediaStressTest implements IDeviceTest, IRemoteTest { + + ITestDevice mTestDevice = null; + private static final String METRICS_RUN_NAME = "VideoRecordingStress"; + + //Max test timeout - 2 hrs + private static final int MAX_TEST_TIMEOUT = 2 * 60 * 60 * 1000; + + // Constants for running the tests + private static final String TEST_CLASS_NAME = + "com.android.mediaframeworktest.stress.MediaRecorderStressTest"; + private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest"; + private static final String TEST_RUNNER_NAME = ".MediaRecorderStressTestRunner"; + + // Constants for parsing the output file + private static final Pattern EXPECTED_LOOP_COUNT_PATTERN = + Pattern.compile("Total number of loops:\\s*(\\d+)"); + private static final Pattern ACTUAL_LOOP_COUNT_PATTERN = + Pattern.compile("No of loop:.*,\\s*(\\d+)\\s*"); + private static final String OUTPUT_PATH = "mediaStressOutput.txt"; + + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + Assert.assertNotNull(mTestDevice); + + IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, + TEST_RUNNER_NAME, mTestDevice.getIDevice()); + runner.setClassName(TEST_CLASS_NAME); + runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS); + + cleanTmpFiles(); + mTestDevice.runInstrumentationTests(runner, listener); + logOutputFile(listener); + cleanTmpFiles(); + } + + /** + * Clean up temp files from test runs + */ + private void cleanTmpFiles() throws DeviceNotAvailableException { + String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + mTestDevice.executeShellCommand(String.format("rm %s/temp*.3gp", extStore)); + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, OUTPUT_PATH)); + } + + /** + * Pull the output file from the device, add it to the logs, and also parse out the relevant + * test metrics and report them. + */ + private void logOutputFile(ITestInvocationListener listener) + throws DeviceNotAvailableException { + File outputFile = null; + InputStreamSource outputSource = null; + try { + outputFile = mTestDevice.pullFileFromExternal(OUTPUT_PATH); + + if (outputFile == null) { + return; + } + + CLog.d("Sending %d byte file %s into the logosphere!", outputFile.length(), outputFile); + outputSource = new FileInputStreamSource(outputFile); + listener.testLog(OUTPUT_PATH, LogDataType.TEXT, outputSource); + parseOutputFile(outputFile, listener); + } finally { + FileUtil.deleteFile(outputFile); + StreamUtil.cancel(outputSource); + } + } + + /** + * Parse the relevant metrics from the Instrumentation test output file + */ + private void parseOutputFile(File outputFile, ITestInvocationListener listener) { + Map<String, String> runMetrics = new HashMap<>(); + Map<String, String> stanzaKeyMap = new HashMap<>(); + stanzaKeyMap.put("testStressRecordVideoAndPlayback1080P", "VideoRecordPlayback1080P"); + stanzaKeyMap.put("testStressRecordVideoAndPlayback720P", "VideoRecordPlayback720P"); + stanzaKeyMap.put("testStressRecordVideoAndPlayback480P", "VideoRecordPlayback480P"); + stanzaKeyMap.put("testStressTimeLapse", "TimeLapseRecord"); + + // try to parse it + String contents; + try { + InputStream dataStream = new FileInputStream(outputFile); + contents = StreamUtil.getStringFromStream(dataStream); + } catch (IOException e) { + CLog.e("IOException while parsing the output file:"); + CLog.e(e); + return; + } + + List<String> lines = Arrays.asList(contents.split("\n")); + ListIterator<String> lineIter = lines.listIterator(); + String line; + while (lineIter.hasNext()) { + line = lineIter.next(); + String key = null; + + if (stanzaKeyMap.containsKey(line)) { + key = stanzaKeyMap.get(line); + } else { + CLog.d("Got unexpected line: %s", line); + continue; + } + + Integer countExpected = getIntFromOutput(lineIter, EXPECTED_LOOP_COUNT_PATTERN); + Integer countActual = getIntFromOutput(lineIter, ACTUAL_LOOP_COUNT_PATTERN); + int value = coalesceLoopCounts(countActual, countExpected); + runMetrics.put(key, Integer.toString(value)); + } + + reportMetrics(listener, runMetrics); + } + + /** + * Report run metrics by creating an empty test run to stick them in + * <p /> + * Exposed for unit testing + */ + void reportMetrics(ITestInvocationListener listener, Map<String, String> metrics) { + // Create an empty testRun to report the parsed runMetrics + CLog.d("About to report metrics: %s", metrics); + listener.testRunStarted(METRICS_RUN_NAME, 0); + listener.testRunEnded(0, metrics); + } + + /** + * Use the provided {@link Pattern} to parse a number out of the output file + */ + private Integer getIntFromOutput(ListIterator<String> lineIter, Pattern numPattern) { + Integer retval = null; + String line = null; + if (lineIter.hasNext()) { + line = lineIter.next(); + Matcher m = numPattern.matcher(line); + if (m.matches()) { + retval = Integer.parseInt(m.group(1)); + } else { + CLog.e("Couldn't match pattern %s against line '%s'", numPattern, line); + } + } else { + CLog.e("Encounted EOF while trying to match pattern %s", numPattern); + } + + return retval; + } + + /** + * Given an actual and an expected iteration count, determine a single metric to report. + */ + private int coalesceLoopCounts(Integer actual, Integer expected) { + if (expected == null || expected <= 0) { + return -1; + } else if (actual == null) { + return expected; + } else { + return actual; + } + } + + @Override + public void setDevice(ITestDevice device) { + mTestDevice = device; + } + + @Override + public ITestDevice getDevice() { + return mTestDevice; + } +} diff --git a/src/com/android/media/tests/MediaTest.java b/src/com/android/media/tests/MediaTest.java new file mode 100644 index 0000000..473f989 --- /dev/null +++ b/src/com/android/media/tests/MediaTest.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import junit.framework.TestCase; + +import java.util.regex.Matcher; + + +public class MediaTest extends TestCase { + private String mPositiveOutput = "testStressAddRemoveEffects diff : 472"; + private String mNegativeOutput = "testStressAddRemoveEffects diff : -123"; + + private String mExpectedTestCaseName = "testStressAddRemoveEffects"; + private String mExpectedPositiveOut = "472"; + private String mExpectedNegativeOut = "-123"; + + public void testVideoEditorPositiveOut() { + Matcher m = VideoEditingMemoryTest.TOTAL_MEM_DIFF_PATTERN.matcher(mPositiveOutput); + assertTrue(m.matches()); + assertEquals("Parse test case name", mExpectedTestCaseName, m.group(1)); + assertEquals("Paser positive out", mExpectedPositiveOut, m.group(2)); + } + + public void testVideoEditorNegativeOut() { + Matcher m = VideoEditingMemoryTest.TOTAL_MEM_DIFF_PATTERN.matcher(mNegativeOutput); + assertTrue(m.matches()); + assertEquals("Parse test case name", mExpectedTestCaseName, m.group(1)); + assertEquals("Paser negative out", mExpectedNegativeOut, m.group(2)); + } +} diff --git a/src/com/android/media/tests/PanoramaBenchMarkTest.java b/src/com/android/media/tests/PanoramaBenchMarkTest.java new file mode 100644 index 0000000..355bbb0 --- /dev/null +++ b/src/com/android/media/tests/PanoramaBenchMarkTest.java @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.IDevice; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; + +import org.junit.Assert; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Standalone panoramic photo processing benchmark test. + */ +public class PanoramaBenchMarkTest implements IDeviceTest, IRemoteTest { + + private ITestDevice mTestDevice = null; + + private static final Pattern ELAPSED_TIME_PATTERN = + Pattern.compile("(Total elapsed time:)\\s+(\\d+\\.\\d*)\\s+(seconds)"); + + private static final String PANORAMA_TEST_KEY = "PanoramaElapsedTime"; + private static final String TEST_TAG = "CameraLatency"; + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + Assert.assertNotNull(mTestDevice); + + String dataStore = mTestDevice.getMountPoint(IDevice.MNT_DATA); + String externalStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + + mTestDevice.executeShellCommand(String.format("chmod 777 %s/local/tmp/panorama_bench", + dataStore)); + + String shellOutput = mTestDevice.executeShellCommand( + String.format("%s/local/tmp/panorama_bench %s/panorama_input/test %s/panorama.ppm", + dataStore, externalStore, externalStore)); + + String[] lines = shellOutput.split("\n"); + + Map<String, String> metrics = new HashMap<String, String>(); + for (String line : lines) { + Matcher m = ELAPSED_TIME_PATTERN.matcher(line.trim()); + if (m.matches()) { + CLog.d(String.format("Found elapsed time \"%s seconds\" from line %s", + m.group(2), line)); + metrics.put(PANORAMA_TEST_KEY, m.group(2)); + break; + } else { + CLog.d(String.format("Unabled to find elapsed time from line: %s", line)); + } + } + + reportMetrics(listener, TEST_TAG, metrics); + cleanupDevice(); + } + + /** + * {@inheritDoc} + */ + @Override + public void setDevice(ITestDevice device) { + mTestDevice = device; + } + + /** + * {@inheritDoc} + */ + @Override + public ITestDevice getDevice() { + return mTestDevice; + } + + /** + * Removes image files used to mock panorama stitching. + * + * @throws DeviceNotAvailableException If the device is unavailable or + * something happened while deleting files + */ + private void cleanupDevice() throws DeviceNotAvailableException { + String externalStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + mTestDevice.executeShellCommand(String.format("rm -r %s/panorama_input", externalStore)); + } + + /** + * Report run metrics by creating an empty test run to stick them in. + * + * @param listener The {@link ITestInvocationListener} of test results + * @param runName The test name + * @param metrics The {@link Map} that contains metrics for the given test + */ + private void reportMetrics(ITestInvocationListener listener, String runName, + Map<String, String> metrics) { + InputStreamSource bugreport = mTestDevice.getBugreport(); + listener.testLog("bugreport", LogDataType.BUGREPORT, bugreport); + bugreport.cancel(); + + CLog.d(String.format("About to report metrics: %s", metrics)); + listener.testRunStarted(runName, 0); + listener.testRunEnded(0, metrics); + } +} diff --git a/src/com/android/media/tests/TestRunHelper.java b/src/com/android/media/tests/TestRunHelper.java new file mode 100644 index 0000000..f752cf3 --- /dev/null +++ b/src/com/android/media/tests/TestRunHelper.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; + +import java.util.HashMap; +import java.util.Map; + +/** Generic helper class for tests */ +public class TestRunHelper { + + private long mTestStartTime = -1; + private long mTestStopTime = -1; + private ITestInvocationListener mListener; + private TestIdentifier mTestId; + + public TestRunHelper(ITestInvocationListener listener, TestIdentifier testId) { + mListener = listener; + mTestId = testId; + } + + public long getTotalTestTime() { + return mTestStopTime - mTestStartTime; + } + + public void reportFailure(String errMsg) { + CLog.e(errMsg); + mListener.testFailed(mTestId, errMsg); + mListener.testEnded(mTestId, new HashMap<String, String>()); + mListener.testRunFailed(errMsg); + } + + /** @param resultDictionary */ + public void endTest(Map<String, String> resultDictionary) { + mTestStopTime = System.currentTimeMillis(); + mListener.testEnded(mTestId, resultDictionary); + mListener.testRunEnded(getTotalTestTime(), resultDictionary); + } + + public void startTest(int numberOfTests) { + mListener.testRunStarted(mTestId.getTestName(), numberOfTests); + mListener.testStarted(mTestId); + mTestStartTime = System.currentTimeMillis(); + } +} diff --git a/src/com/android/media/tests/VideoEditingMemoryTest.java b/src/com/android/media/tests/VideoEditingMemoryTest.java new file mode 100644 index 0000000..bb58df2 --- /dev/null +++ b/src/com/android/media/tests/VideoEditingMemoryTest.java @@ -0,0 +1,268 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.IDevice; +import com.android.ddmlib.Log; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.android.tradefed.config.Option; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.result.BugreportCollector; +import com.android.tradefed.result.BugreportCollector.Freq; +import com.android.tradefed.result.BugreportCollector.Noun; +import com.android.tradefed.result.BugreportCollector.Relation; +import com.android.tradefed.result.FileInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.StreamUtil; + +import org.junit.Assert; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Runs the Video Editing Framework Memory Tests. This test exercise the basic + * functionality of video editing test and capture the memory usage. The memory + * usage test out is saved in /sdcard/VideoEditorStressMemOutput.txt and + * VideoEditorMediaServerMemoryLog.txt. + * <p/> + * Note that this test will not run properly unless /sdcard is mounted and + * writable. + */ +public class VideoEditingMemoryTest implements IDeviceTest, IRemoteTest { + private static final String LOG_TAG = "VideoEditorMemoryTest"; + + ITestDevice mTestDevice = null; + + // Constants for running the tests + private static final String TEST_CLASS_NAME = + "com.android.mediaframeworktest.stress.VideoEditorStressTest"; + private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest"; + private static final String TEST_RUNNER_NAME = ".MediaPlayerStressTestRunner"; + + //Max test timeout - 3 hrs + private static final int MAX_TEST_TIMEOUT = 3 * 60 * 60 * 1000; + + /* + * Pattern to find the test case name and test result. + * Example of a matching line: + * testStressAddRemoveEffects total diff = 0 + * The first string 'testStressAddRemoveEffects' represent the dashboard key and + * the last string represent the test result. + */ + public static final Pattern TOTAL_MEM_DIFF_PATTERN = + Pattern.compile("(.+?)\\s.*diff.*\\s(-?\\d+)"); + + public Map<String, String> mRunMetrics = new HashMap<>(); + public Map<String, String> mKeyMap = new HashMap<>(); + + @Option(name = "getHeapDump", description = "Collect the heap") + private boolean mGetHeapDump = false; + + public VideoEditingMemoryTest() { + mKeyMap.put("VideoEditorStressMemOutput.txt", "VideoEditorMemory"); + mKeyMap.put("VideoEditorMediaServerMemoryLog.txt", + "VideoEditorMemoryMediaServer"); + } + + + @Override + public void run(ITestInvocationListener listener) + throws DeviceNotAvailableException { + Assert.assertNotNull(mTestDevice); + + IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner( + TEST_PACKAGE_NAME, TEST_RUNNER_NAME, mTestDevice.getIDevice()); + runner.setClassName(TEST_CLASS_NAME); + runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS); + if (mGetHeapDump) { + runner.addInstrumentationArg("get_heap_dump", "getNativeHeap"); + } + + BugreportCollector bugListener = new BugreportCollector(listener, + mTestDevice); + bugListener.addPredicate(new BugreportCollector.Predicate( + Relation.AFTER, Freq.EACH, Noun.TESTRUN)); + + mTestDevice.runInstrumentationTests(runner, bugListener); + + logOutputFiles(listener); + cleanResultFile(); + } + + /** + * Clean up the test result file from test run + */ + private void cleanResultFile() throws DeviceNotAvailableException { + String extStore = + mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + for(String outFile : mKeyMap.keySet()) { + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, + outFile)); + } + if (mGetHeapDump) { + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, + "*.dump")); + } + } + + private void uploadHeapDumpFiles(ITestInvocationListener listener) + throws DeviceNotAvailableException { + // Pull and upload the heap dump output files. + InputStreamSource outputSource = null; + File outputFile = null; + + String extStore = + mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + + String out = mTestDevice.executeShellCommand(String.format("ls %s/%s", + extStore, "*.dump")); + String heapOutputFiles[] = out.split("\n"); + + for (String heapOutputFile : heapOutputFiles) { + try { + outputFile = mTestDevice.pullFile(heapOutputFile.trim()); + if (outputFile == null) { + continue; + } + outputSource = new FileInputStreamSource(outputFile); + listener.testLog(heapOutputFile, LogDataType.TEXT, outputSource); + } finally { + FileUtil.deleteFile(outputFile); + StreamUtil.cancel(outputSource); + } + } + } + + /** + * Pull the output files from the device, add it to the logs, and also parse + * out the relevant test metrics and report them. + */ + private void logOutputFiles(ITestInvocationListener listener) + throws DeviceNotAvailableException { + File outputFile = null; + InputStreamSource outputSource = null; + + if (mGetHeapDump) { + // Upload all the heap dump files. + uploadHeapDumpFiles(listener); + } + for (String resultFile : mKeyMap.keySet()) { + try { + outputFile = mTestDevice.pullFileFromExternal(resultFile); + + if (outputFile == null) { + return; + } + + // Upload a verbatim copy of the output file + Log.d(LOG_TAG, String.format( + "Sending %d byte file %s into the logosphere!", + outputFile.length(), outputFile)); + outputSource = new FileInputStreamSource(outputFile); + listener.testLog(resultFile, LogDataType.TEXT, outputSource); + + // Parse the output file to upload aggregated metrics + parseOutputFile(new FileInputStream(outputFile), listener, resultFile); + } catch (IOException e) { + Log.e( + LOG_TAG, + String.format("IOException while reading or parsing output file: %s", e)); + } finally { + FileUtil.deleteFile(outputFile); + StreamUtil.cancel(outputSource); + } + } + } + + /** + * Parse the relevant metrics from the Instrumentation test output file + */ + private void parseOutputFile(InputStream dataStream, + ITestInvocationListener listener, String outputFile) { + + // try to parse it + String contents; + try { + contents = StreamUtil.getStringFromStream(dataStream); + } catch (IOException e) { + Log.e(LOG_TAG, String.format( + "Got IOException during test processing: %s", e)); + return; + } + + List<String> lines = Arrays.asList(contents.split("\n")); + ListIterator<String> lineIter = lines.listIterator(); + + String line; + String key; + String memOut; + + while (lineIter.hasNext()){ + line = lineIter.next(); + + // Look for the total diff + Matcher m = TOTAL_MEM_DIFF_PATTERN.matcher(line); + if (m.matches()){ + //First group match with the test key name. + key = m.group(1); + //Second group match witht the test result. + memOut = m.group(2); + mRunMetrics.put(key, memOut); + } + } + reportMetrics(listener, outputFile); + } + + /** + * Report run metrics by creating an empty test run to stick them in + * <p /> + * Exposed for unit testing + */ + void reportMetrics(ITestInvocationListener listener, String outputFile) { + Log.d(LOG_TAG, String.format("About to report metrics: %s", mRunMetrics)); + listener.testRunStarted(mKeyMap.get(outputFile), 0); + listener.testRunEnded(0, mRunMetrics); + } + + @Override + public void setDevice(ITestDevice device) { + mTestDevice = device; + } + + @Override + public ITestDevice getDevice() { + return mTestDevice; + } +} diff --git a/src/com/android/media/tests/VideoEditingPerformanceTest.java b/src/com/android/media/tests/VideoEditingPerformanceTest.java new file mode 100644 index 0000000..046403e --- /dev/null +++ b/src/com/android/media/tests/VideoEditingPerformanceTest.java @@ -0,0 +1,257 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.IDevice; +import com.android.ddmlib.Log; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.result.BugreportCollector; +import com.android.tradefed.result.BugreportCollector.Freq; +import com.android.tradefed.result.BugreportCollector.Noun; +import com.android.tradefed.result.BugreportCollector.Relation; +import com.android.tradefed.result.FileInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.InputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.RegexTrie; +import com.android.tradefed.util.StreamUtil; + +import org.junit.Assert; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Runs the Video Editing Framework Performance Test.The performance test result + * is saved in /sdcard/VideoEditorPerformance.txt + * <p/> + * Note that this test will not run properly unless /sdcard is mounted and + * writable. + */ +public class VideoEditingPerformanceTest implements IDeviceTest, IRemoteTest { + private static final String LOG_TAG = "VideoEditingPerformanceTest"; + + ITestDevice mTestDevice = null; + + private static final String METRICS_RUN_NAME = "VideoEditor"; + + //Max test timeout - 3 hrs + private static final int MAX_TEST_TIMEOUT = 3 * 60 * 60 * 1000; + + // Constants for running the tests + private static final String TEST_CLASS_NAME = + "com.android.mediaframeworktest.performance.VideoEditorPerformance"; + private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest"; + private static final String TEST_RUNNER_NAME = ".MediaFrameworkPerfTestRunner"; + + private static final String OUTPUT_PATH = "VideoEditorPerformance.txt"; + + private final RegexTrie<String> mPatternMap = new RegexTrie<>(); + + public VideoEditingPerformanceTest() { + mPatternMap.put("ImageItemCreate", + "^.*Time taken to Create Media Image Item :(\\d+)"); + mPatternMap.put("mageItemAdd", + "^.*Time taken to add Media Image Item :(\\d+)"); + mPatternMap.put("ImageItemRemove", + "^.*Time taken to remove Media Image Item :(\\d+)"); + mPatternMap.put("ImageItemCreate640x480", + "^.*Time taken to Create Media Image Item.*640x480.*:(\\d+)"); + mPatternMap.put("ImageItemAdd640x480", + "^.*Time taken to add Media Image Item.*640x480.*:(\\d+)"); + mPatternMap.put("ImageItemRemove640x480", + "^.*Time taken to remove Media Image Item.*640x480.*:(\\d+)"); + mPatternMap.put("CrossFadeTransitionCreate", + "^.*Time taken to Create CrossFade Transition :(\\d+)"); + mPatternMap.put("CrossFadeTransitionAdd", + "^.*Time taken to add CrossFade Transition :(\\d+)"); + mPatternMap.put("CrossFadeTransitionRemove", + "^.*Time taken to remove CrossFade Transition :(\\d+)"); + mPatternMap.put("VideoItemCreate", + "^.*Time taken to Create Media Video Item :(\\d+)"); + mPatternMap.put("VideoItemAdd", + "^.*Time taken to Add Media Video Item :(\\d+)"); + mPatternMap.put("VideoItemRemove", + "^.*Time taken to remove Media Video Item :(\\d+)"); + mPatternMap.put("EffectOverlappingTransition", + "^.*Time taken to testPerformanceEffectOverlappingTransition :(\\d+.\\d+)"); + mPatternMap.put("ExportStoryboard", + "^.*Time taken to do ONE export of storyboard duration 69000 is :(\\d+)"); + mPatternMap.put("PreviewWithTransition", + "^.*Time taken to Generate Preview with transition :(\\d+.\\d+)"); + mPatternMap.put("OverlayCreate", + "^.*Time taken to add & create Overlay :(\\d+)"); + mPatternMap.put("OverlayRemove", + "^.*Time taken to remove Overlay :(\\d+)"); + mPatternMap.put("GetVideoThumbnails", + "^.*Duration taken to get Video Thumbnails :(\\d+)"); + mPatternMap.put("TransitionWithEffectOverlapping", + "^.*Time taken to TransitionWithEffectOverlapping :(\\d+.\\d+)"); + mPatternMap.put("MediaPropertiesGet", + "^.*Time taken to get Media Properties :(\\d+)"); + mPatternMap.put("AACLCAdd", + "^.*Time taken for 1st Audio Track.*AACLC.*:(\\d+)"); + mPatternMap.put("AMRNBAdd", + "^.*Time taken for 2nd Audio Track.*AMRNB.*:(\\d+)"); + mPatternMap.put("KenBurnGeneration", + "^.*Time taken to Generate KenBurn Effect :(\\d+.\\d+)"); + mPatternMap.put("ThumbnailsGeneration", + "^.*Time taken Thumbnail generation :(\\d+.\\d+)"); + mPatternMap.put("H264ThumbnailGeneration", + "^.*Time taken for Thumbnail generation :(\\d+.\\d+)"); + } + + @Override + public void run(ITestInvocationListener listener) + throws DeviceNotAvailableException { + Assert.assertNotNull(mTestDevice); + + IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner( + TEST_PACKAGE_NAME, TEST_RUNNER_NAME, mTestDevice.getIDevice()); + runner.setClassName(TEST_CLASS_NAME); + runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS); + + BugreportCollector bugListener = new BugreportCollector(listener, + mTestDevice); + bugListener.addPredicate(new BugreportCollector.Predicate( + Relation.AFTER, Freq.EACH, Noun.TESTRUN)); + + mTestDevice.runInstrumentationTests(runner, bugListener); + + logOutputFiles(listener); + cleanResultFile(); + } + + /** + * Clean up the test result file from test run + */ + private void cleanResultFile() throws DeviceNotAvailableException { + String extStore = + mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); + mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, OUTPUT_PATH)); + } + + /** + * Pull the output files from the device, add it to the logs, and also parse + * out the relevant test metrics and report them. + */ + private void logOutputFiles(ITestInvocationListener listener) + throws DeviceNotAvailableException { + File outputFile = null; + InputStreamSource outputSource = null; + try { + outputFile = mTestDevice.pullFileFromExternal(OUTPUT_PATH); + + if (outputFile == null) { + return; + } + + // Upload a verbatim copy of the output file + Log.d(LOG_TAG, String.format( + "Sending %d byte file %s into the logosphere!", + outputFile.length(), outputFile)); + outputSource = new FileInputStreamSource(outputFile); + listener.testLog(OUTPUT_PATH, LogDataType.TEXT, outputSource); + + // Parse the output file to upload aggregated metrics + parseOutputFile(new FileInputStream(outputFile), listener); + } catch (IOException e) { + Log.e(LOG_TAG, String.format( + "IOException while reading or parsing output file: %s", e)); + } finally { + FileUtil.deleteFile(outputFile); + StreamUtil.cancel(outputSource); + } + } + + /** + * Parse the relevant metrics from the Instrumentation test output file + */ + private void parseOutputFile(InputStream dataStream, + ITestInvocationListener listener) { + + Map<String, String> runMetrics = new HashMap<>(); + + // try to parse it + String contents; + try { + contents = StreamUtil.getStringFromStream(dataStream); + } catch (IOException e) { + Log.e(LOG_TAG, String.format( + "Got IOException during test processing: %s", e)); + return; + } + + List<String> lines = Arrays.asList(contents.split("\n")); + ListIterator<String> lineIter = lines.listIterator(); + String line; + while (lineIter.hasNext()) { + line = lineIter.next(); + List<List<String>> capture = new ArrayList<>(1); + String key = mPatternMap.retrieve(capture, line); + if (key != null) { + Log.d(LOG_TAG, String.format("Got '%s' and captures '%s'", key, + capture.toString())); + } else if (line.isEmpty()) { + // ignore + continue; + } else { + Log.e(LOG_TAG, String.format("Got unmatched line: %s", line)); + continue; + } + runMetrics.put(key, capture.get(0).get(0)); + } + reportMetrics(listener, runMetrics); + } + + /** + * Report run metrics by creating an empty test run to stick them in + * <p /> + * Exposed for unit testing + */ + void reportMetrics(ITestInvocationListener listener, + Map<String, String> metrics) { + Log.d(LOG_TAG, String.format("About to report metrics: %s", metrics)); + listener.testRunStarted(METRICS_RUN_NAME, 0); + listener.testRunEnded(0, metrics); + } + + @Override + public void setDevice(ITestDevice device) { + mTestDevice = device; + } + + @Override + public ITestDevice getDevice() { + return mTestDevice; + } +} diff --git a/src/com/android/media/tests/VideoMultimeterRunner.java b/src/com/android/media/tests/VideoMultimeterRunner.java new file mode 100644 index 0000000..13fd4d1 --- /dev/null +++ b/src/com/android/media/tests/VideoMultimeterRunner.java @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.Option; +import com.android.tradefed.config.Option.Importance; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.CommandResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Semaphore; + +/** + * A harness that test video playback with multiple devices and reports result. + */ +public class VideoMultimeterRunner extends VideoMultimeterTest + implements IDeviceTest, IRemoteTest { + + @Option(name = "robot-util-path", description = "path for robot control util", + importance = Importance.ALWAYS, mandatory = true) + String mRobotUtilPath = "/tmp/robot_util.sh"; + + @Option(name = "device-map", description = + "Device serials map to location and audio input. May be repeated", + importance = Importance.ALWAYS) + Map<String, String> mDeviceMap = new HashMap<String, String>(); + + @Option(name = "calibration-map", description = + "Device serials map to calibration values. May be repeated", + importance = Importance.ALWAYS) + Map<String, String> mCalibrationMap = new HashMap<String, String>(); + + static final long ROBOT_TIMEOUT_MS = 60 * 1000; + + static final Semaphore runToken = new Semaphore(1); + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) + throws DeviceNotAvailableException { + long durationMs = 0; + TestIdentifier testId = new TestIdentifier(getClass() + .getCanonicalName(), RUN_KEY); + + listener.testRunStarted(RUN_KEY, 0); + listener.testStarted(testId); + + long testStartTime = System.currentTimeMillis(); + Map<String, String> metrics = new HashMap<String, String>(); + + try { + CLog.v("Waiting to acquire run token"); + runToken.acquire(); + + String deviceSerial = getDevice().getSerialNumber(); + + String calibrationValue = (mCalibrationMap.containsKey(deviceSerial) ? + mCalibrationMap.get(deviceSerial) : null); + if (mDebugWithoutHardware + || (moveArm(deviceSerial) && setupTestEnv(calibrationValue))) { + runMultimeterTest(listener, metrics); + } else { + listener.testFailed(testId, "Failed to set up environment"); + } + } catch (InterruptedException e) { + CLog.d("Acquire run token interrupted"); + listener.testFailed(testId, "Failed to acquire run token"); + } finally { + runToken.release(); + listener.testEnded(testId, metrics); + durationMs = System.currentTimeMillis() - testStartTime; + listener.testRunEnded(durationMs, metrics); + } + } + + protected boolean moveArm(String deviceSerial) { + if (mDeviceMap.containsKey(deviceSerial)) { + CLog.v("Moving robot arm to device " + deviceSerial); + CommandResult cr = getRunUtil().runTimedCmd( + ROBOT_TIMEOUT_MS, mRobotUtilPath, mDeviceMap.get(deviceSerial)); + CLog.v(cr.getStdout()); + return true; + } else { + CLog.e("Cannot find device in map, test failed"); + return false; + } + } +} diff --git a/src/com/android/media/tests/VideoMultimeterTest.java b/src/com/android/media/tests/VideoMultimeterTest.java new file mode 100644 index 0000000..038803f --- /dev/null +++ b/src/com/android/media/tests/VideoMultimeterTest.java @@ -0,0 +1,490 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.media.tests; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.android.tradefed.config.Option; +import com.android.tradefed.config.Option.Importance; +import com.android.tradefed.device.CollectingOutputReceiver; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.result.ByteArrayInputStreamSource; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.testtype.IDeviceTest; +import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.util.CommandResult; +import com.android.tradefed.util.IRunUtil; +import com.android.tradefed.util.RunUtil; + +import org.junit.Assert; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A harness that test video playback and reports result. + */ +public class VideoMultimeterTest implements IDeviceTest, IRemoteTest { + + static final String RUN_KEY = "video_multimeter"; + + @Option(name = "multimeter-util-path", description = "path for multimeter control util", + importance = Importance.ALWAYS) + String mMeterUtilPath = "/tmp/util.sh"; + + @Option(name = "start-video-cmd", description = "adb shell command to start video playback; " + + "use '%s' as placeholder for media source filename", importance = Importance.ALWAYS) + String mCmdStartVideo = "am instrument -w -r -e media-file" + + " \"%s\" -e class com.android.mediaframeworktest.stress.MediaPlayerStressTest" + + " com.android.mediaframeworktest/.MediaPlayerStressTestRunner"; + + @Option(name = "stop-video-cmd", description = "adb shell command to stop video playback", + importance = Importance.ALWAYS) + String mCmdStopVideo = "am force-stop com.android.mediaframeworktest"; + + @Option(name="video-spec", description= + "Comma deliminated information for test video files with the following format: " + + "video_filename, reporting_key_prefix, fps, duration(in sec) " + + "May be repeated for test with multiple files.") + private Collection<String> mVideoSpecs = new ArrayList<>(); + + @Option(name="wait-time-between-runs", description= + "wait time between two test video measurements, in millisecond") + private long mWaitTimeBetweenRuns = 3 * 60 * 1000; + + @Option(name="calibration-video", description= + "filename of calibration video") + private String mCaliVideoDevicePath = "video_cali.mp4"; + + @Option( + name = "debug-without-hardware", + description = "Use option to debug test without having specialized hardware", + importance = Importance.NEVER, + mandatory = false + ) + protected boolean mDebugWithoutHardware = false; + + static final String ROTATE_LANDSCAPE = "content insert --uri content://settings/system" + + " --bind name:s:user_rotation --bind value:i:1"; + + // Max number of trailing frames to trim + static final int TRAILING_FRAMES_MAX = 3; + // Min threshold for duration of trailing frames + static final long FRAME_DURATION_THRESHOLD_US = 500 * 1000; // 0.5s + + static final String CMD_GET_FRAMERATE_STATE = "GETF"; + static final String CMD_START_CALIBRATION = "STAC"; + static final String CMD_SET_CALIBRATION_VALS = "SETCAL"; + static final String CMD_STOP_CALIBRATION = "STOC"; + static final String CMD_START_MEASUREMENT = "STAM"; + static final String CMD_STOP_MEASUREMENT = "STOM"; + static final String CMD_GET_NUM_FRAMES = "GETN"; + static final String CMD_GET_ALL_DATA = "GETD"; + + static final long DEVICE_SYNC_TIME_MS = 30 * 1000; + static final long CALIBRATION_TIMEOUT_MS = 30 * 1000; + static final long COMMAND_TIMEOUT_MS = 5 * 1000; + static final long GETDATA_TIMEOUT_MS = 10 * 60 * 1000; + + // Regex for: "OK (time); (frame duration); (marker color); (total dropped frames)" + static final String VIDEO_FRAME_DATA_PATTERN = "OK\\s+\\d+;\\s*(-?\\d+);\\s*[a-z]+;\\s*(\\d+)"; + + // Regex for: "OK (time); (frame duration); (marker color); (total dropped frames); (lipsync)" + // results in: $1 == ts, $2 == lipsync + static final String LIPSYNC_DATA_PATTERN = + "OK\\s+(\\d+);\\s*\\d+;\\s*[a-z]+;\\s*\\d+;\\s*(-?\\d+)"; + // ts dur color missed latency + static final int LIPSYNC_SIGNAL = 2000000; // every 2 seconds + static final int LIPSYNC_SIGNAL_MIN = 1500000; // must be at least 1.5 seconds after prev + + ITestDevice mDevice; + + /** + * {@inheritDoc} + */ + @Override + public void setDevice(ITestDevice device) { + mDevice = device; + } + + /** + * {@inheritDoc} + */ + @Override + public ITestDevice getDevice() { + return mDevice; + } + + private void rotateScreen() throws DeviceNotAvailableException { + // rotate to landscape mode, except for manta + if (!getDevice().getProductType().contains("manta")) { + getDevice().executeShellCommand(ROTATE_LANDSCAPE); + } + } + + protected boolean setupTestEnv() throws DeviceNotAvailableException { + return setupTestEnv(null); + } + + protected void startPlayback(final String videoPath) { + new Thread() { + @Override + public void run() { + try { + CollectingOutputReceiver receiver = new CollectingOutputReceiver(); + getDevice().executeShellCommand(String.format( + mCmdStartVideo, videoPath), + receiver, 1L, TimeUnit.SECONDS, 0); + } catch (DeviceNotAvailableException e) { + CLog.e(e.getMessage()); + } + } + }.start(); + } + + /** + * Perform calibration process for video multimeter + * + * @return boolean whether calibration succeeds + * @throws DeviceNotAvailableException + */ + protected boolean doCalibration() throws DeviceNotAvailableException { + // play calibration video + startPlayback(mCaliVideoDevicePath); + getRunUtil().sleep(3 * 1000); + rotateScreen(); + getRunUtil().sleep(1 * 1000); + CommandResult cr = getRunUtil().runTimedCmd( + COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_START_CALIBRATION); + CLog.i("Starting calibration: " + cr.getStdout()); + // check whether multimeter is calibrated + boolean isCalibrated = false; + long calibrationStartTime = System.currentTimeMillis(); + while (!isCalibrated && + System.currentTimeMillis() - calibrationStartTime <= CALIBRATION_TIMEOUT_MS) { + getRunUtil().sleep(1 * 1000); + cr = getRunUtil().runTimedCmd(2 * 1000, mMeterUtilPath, CMD_GET_FRAMERATE_STATE); + if (cr.getStdout().contains("calib0")) { + isCalibrated = true; + } + } + if (!isCalibrated) { + // stop calibration if time out + cr = getRunUtil().runTimedCmd( + COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_STOP_CALIBRATION); + CLog.e("Calibration timed out."); + } else { + CLog.i("Calibration succeeds."); + } + getDevice().executeShellCommand(mCmdStopVideo); + return isCalibrated; + } + + protected boolean setupTestEnv(String caliValues) throws DeviceNotAvailableException { + getRunUtil().sleep(DEVICE_SYNC_TIME_MS); + CommandResult cr = getRunUtil().runTimedCmd( + COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_STOP_MEASUREMENT); + + getDevice().setDate(new Date()); + CLog.i("syncing device time to host time"); + getRunUtil().sleep(3 * 1000); + + // TODO: need a better way to clear old data + // start and stop to clear old data + cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_START_MEASUREMENT); + getRunUtil().sleep(3 * 1000); + cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_STOP_MEASUREMENT); + getRunUtil().sleep(3 * 1000); + CLog.i("Stopping measurement: " + cr.getStdout()); + + if (caliValues == null) { + return doCalibration(); + } else { + CLog.i("Setting calibration values: " + caliValues); + final String calibrationValues = CMD_SET_CALIBRATION_VALS + " " + caliValues; + cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, calibrationValues); + final String response = mDebugWithoutHardware ? "OK" : cr.getStdout(); + if (response != null && response.startsWith("OK")) { + CLog.i("Calibration values are set to: " + caliValues); + return true; + } else { + CLog.e("Failed to set calibration values: " + cr.getStdout()); + return false; + } + } + } + + private void doMeasurement(final String testVideoPath, long durationSecond) + throws DeviceNotAvailableException { + CommandResult cr; + getDevice().clearErrorDialogs(); + getRunUtil().sleep(mWaitTimeBetweenRuns); + + // play test video + startPlayback(testVideoPath); + + getRunUtil().sleep(3 * 1000); + + rotateScreen(); + getRunUtil().sleep(1 * 1000); + cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_START_MEASUREMENT); + CLog.i("Starting measurement: " + cr.getStdout()); + + // end measurement + getRunUtil().sleep(durationSecond * 1000); + + cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_STOP_MEASUREMENT); + CLog.i("Stopping measurement: " + cr.getStdout()); + if (cr == null || !cr.getStdout().contains("OK")) { + cr = getRunUtil().runTimedCmd( + COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_STOP_MEASUREMENT); + CLog.i("Retry - Stopping measurement: " + cr.getStdout()); + } + + getDevice().executeShellCommand(mCmdStopVideo); + getDevice().clearErrorDialogs(); + } + + private Map<String, String> getResult(ITestInvocationListener listener, + Map<String, String> metrics, String keyprefix, float fps, boolean lipsync) { + CommandResult cr; + + // get number of results + getRunUtil().sleep(5 * 1000); + cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_GET_NUM_FRAMES); + String frameNum = cr.getStdout(); + + CLog.i("== Video Multimeter Result '%s' ==", keyprefix); + CLog.i("Number of results: " + frameNum); + + String nrOfDataPointsStr = extractNumberOfCollectedDataPoints(frameNum); + metrics.put(keyprefix + "frame_captured", nrOfDataPointsStr); + + long nrOfDataPoints = Long.parseLong(nrOfDataPointsStr); + + Assert.assertTrue("Multimeter did not collect any data for " + keyprefix, + nrOfDataPoints > 0); + + CLog.i("Captured frames: " + nrOfDataPointsStr); + + // get all results from multimeter and write to output file + cr = getRunUtil().runTimedCmd(GETDATA_TIMEOUT_MS, mMeterUtilPath, CMD_GET_ALL_DATA); + String allData = cr.getStdout(); + listener.testLog( + keyprefix, LogDataType.TEXT, new ByteArrayInputStreamSource(allData.getBytes())); + + // parse results + return parseResult(metrics, nrOfDataPoints, allData, keyprefix, fps, lipsync); + } + + private String extractNumberOfCollectedDataPoints(String numFrames) { + // Create pattern that matches string like "OK 14132" capturing the + // number of data points. + Pattern p = Pattern.compile("OK\\s+(\\d+)$"); + Matcher m = p.matcher(numFrames.trim()); + + String frameCapturedStr = "0"; + if (m.matches()) { + frameCapturedStr = m.group(1); + } + + return frameCapturedStr; + } + + protected void runMultimeterTest(ITestInvocationListener listener, + Map<String,String> metrics) throws DeviceNotAvailableException { + for (String videoSpec : mVideoSpecs) { + String[] videoInfo = videoSpec.split(","); + String filename = videoInfo[0].trim(); + String keyPrefix = videoInfo[1].trim(); + float fps = Float.parseFloat(videoInfo[2].trim()); + long duration = Long.parseLong(videoInfo[3].trim()); + doMeasurement(filename, duration); + metrics = getResult(listener, metrics, keyPrefix, fps, true); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) + throws DeviceNotAvailableException { + TestIdentifier testId = new TestIdentifier(getClass() + .getCanonicalName(), RUN_KEY); + + listener.testRunStarted(RUN_KEY, 0); + listener.testStarted(testId); + + long testStartTime = System.currentTimeMillis(); + Map<String, String> metrics = new HashMap<>(); + + if (setupTestEnv()) { + runMultimeterTest(listener, metrics); + } + + long durationMs = System.currentTimeMillis() - testStartTime; + listener.testEnded(testId, metrics); + listener.testRunEnded(durationMs, metrics); + } + + /** + * Parse Multimeter result. + * + * @param result + * @return a {@link HashMap} that contains metrics keys and results + */ + private Map<String, String> parseResult(Map<String, String> metrics, + long frameCaptured, String result, String keyprefix, float fps, + boolean lipsync) { + final int MISSING_FRAME_CEILING = 5; //5+ frames missing count the same + final double[] MISSING_FRAME_WEIGHT = {0.0, 1.0, 2.5, 5.0, 6.25, 8.0}; + + // Get total captured frames and calculate smoothness and freezing score + // format: "OK (time); (frame duration); (marker color); (total dropped frames)" + Pattern p = Pattern.compile(VIDEO_FRAME_DATA_PATTERN); + Matcher m = null; + String[] lines = result.split(System.getProperty("line.separator")); + String totalDropFrame = "-1"; + String lastDropFrame = "0"; + long frameCount = 0; + long consecutiveDropFrame = 0; + double freezingPenalty = 0.0; + long frameDuration = 0; + double offByOne = 0; + double offByMultiple = 0; + double expectedFrameDurationInUs = 1000000.0 / fps; + for (int i = 0; i < lines.length; i++) { + m = p.matcher(lines[i].trim()); + if (m.matches()) { + frameDuration = Long.parseLong(m.group(1)); + // frameDuration = -1 indicates dropped frame + if (frameDuration > 0) { + frameCount++; + } + totalDropFrame = m.group(2); + // trim the last few data points if needed + if (frameCount >= frameCaptured - TRAILING_FRAMES_MAX - 1 && + frameDuration > FRAME_DURATION_THRESHOLD_US) { + metrics.put(keyprefix + "frame_captured", String.valueOf(frameCount)); + break; + } + if (lastDropFrame.equals(totalDropFrame)) { + if (consecutiveDropFrame > 0) { + freezingPenalty += MISSING_FRAME_WEIGHT[(int) (Math.min(consecutiveDropFrame, + MISSING_FRAME_CEILING))] * consecutiveDropFrame; + consecutiveDropFrame = 0; + } + } else { + consecutiveDropFrame++; + } + lastDropFrame = totalDropFrame; + + if (frameDuration < expectedFrameDurationInUs * 0.5) { + offByOne++; + } else if (frameDuration > expectedFrameDurationInUs * 1.5) { + if (frameDuration < expectedFrameDurationInUs * 2.5) { + offByOne++; + } else { + offByMultiple++; + } + } + } + } + if (totalDropFrame.equals("-1")) { + // no matching result found + CLog.w("No result found for " + keyprefix); + return metrics; + } else { + metrics.put(keyprefix + "frame_drop", totalDropFrame); + CLog.i("Dropped frames: " + totalDropFrame); + } + double smoothnessScore = 100.0 - (offByOne / frameCaptured) * 100.0 - + (offByMultiple / frameCaptured) * 300.0; + metrics.put(keyprefix + "smoothness", String.valueOf(smoothnessScore)); + CLog.i("Off by one frame: " + offByOne); + CLog.i("Off by multiple frames: " + offByMultiple); + CLog.i("Smoothness score: " + smoothnessScore); + + double freezingScore = 100.0 - 100.0 * freezingPenalty / frameCaptured; + metrics.put(keyprefix + "freezing", String.valueOf(freezingScore)); + CLog.i("Freezing score: " + freezingScore); + + // parse lipsync results (the audio and video synchronization offset) + // format: "OK (time); (frame duration); (marker color); (total dropped frames); (lipsync)" + if (lipsync) { + ArrayList<Integer> lipsyncVals = new ArrayList<>(); + StringBuilder lipsyncValsStr = new StringBuilder("["); + long lipsyncSum = 0; + int lipSyncLastTime = -1; + + Pattern pLip = Pattern.compile(LIPSYNC_DATA_PATTERN); + for (int i = 0; i < lines.length; i++) { + m = pLip.matcher(lines[i].trim()); + if (m.matches()) { + int lipSyncTime = Integer.parseInt(m.group(1)); + int lipSyncVal = Integer.parseInt(m.group(2)); + if (lipSyncLastTime != -1) { + if ((lipSyncTime - lipSyncLastTime) < LIPSYNC_SIGNAL_MIN) { + continue; // ignore the early/spurious one + } + } + lipSyncLastTime = lipSyncTime; + + lipsyncVals.add(lipSyncVal); + lipsyncValsStr.append(lipSyncVal); + lipsyncValsStr.append(", "); + lipsyncSum += lipSyncVal; + } + } + if (lipsyncVals.size() > 0) { + lipsyncValsStr.append("]"); + CLog.i("Lipsync values: " + lipsyncValsStr); + Collections.sort(lipsyncVals); + int lipsyncCount = lipsyncVals.size(); + int minLipsync = lipsyncVals.get(0); + int maxLipsync = lipsyncVals.get(lipsyncCount - 1); + metrics.put(keyprefix + "lipsync_count", String.valueOf(lipsyncCount)); + CLog.i("Lipsync Count: " + lipsyncCount); + metrics.put(keyprefix + "lipsync_min", String.valueOf(lipsyncVals.get(0))); + CLog.i("Lipsync Min: " + minLipsync); + metrics.put(keyprefix + "lipsync_max", String.valueOf(maxLipsync)); + CLog.i("Lipsync Max: " + maxLipsync); + double meanLipSync = (double) lipsyncSum / lipsyncCount; + metrics.put(keyprefix + "lipsync_mean", String.valueOf(meanLipSync)); + CLog.i("Lipsync Mean: " + meanLipSync); + } else { + CLog.w("Lipsync value not found in result."); + } + } + CLog.i("== End ==", keyprefix); + return metrics; + } + + protected IRunUtil getRunUtil() { + return RunUtil.getDefault(); + } +} |